diff --git a/.github/workflows/generate-changelog.yaml b/.github/workflows/generate-changelog.yaml index 71592e4d46..a0538e4eac 100644 --- a/.github/workflows/generate-changelog.yaml +++ b/.github/workflows/generate-changelog.yaml @@ -19,8 +19,8 @@ jobs: with: ref: master - - name: Download changelog - run: ./build.cmd docs-update --depth 1 --preview + - name: Fetch changelog + run: ./build.cmd docs-fetch --depth 1 --preview --force-clone env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -28,8 +28,8 @@ jobs: uses: JamesIves/github-pages-deploy-action@3.7.1 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: docs-changelog-details - FOLDER: docs/_changelog/details + BRANCH: docs-changelog + FOLDER: docs/_changelog GIT_CONFIG_NAME: Andrey Akinshin GIT_CONFIG_EMAIL: andrey.akinshin@gmail.com - CLEAN: true \ No newline at end of file + CLEAN: true diff --git a/.github/workflows/generate-gh-pages.yaml b/.github/workflows/generate-gh-pages.yaml index cb01860995..8983ed711f 100644 --- a/.github/workflows/generate-gh-pages.yaml +++ b/.github/workflows/generate-gh-pages.yaml @@ -22,8 +22,8 @@ jobs: - name: Build BenchmarkDotNet run: ./build.cmd build - - name: Download changelog - run: ./build.cmd docs-update --depth 1 --preview + - name: Fetch changelog + run: ./build.cmd docs-fetch --depth 1 --preview --force-clone env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -31,7 +31,7 @@ jobs: run: ./build.cmd docs-build - name: Upload Artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 with: name: site path: docs/_site @@ -48,9 +48,13 @@ jobs: ref: docs-stable - name: Download Artifacts - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v4 with: name: site + path: site + + - name: Print file tree + run: tree $GITHUB_WORKSPACE - name: Deploy documentation uses: JamesIves/github-pages-deploy-action@3.7.1 diff --git a/.github/workflows/publish-nightly.yaml b/.github/workflows/publish-nightly.yaml index 3a3dc0bc9b..01f439f41e 100644 --- a/.github/workflows/publish-nightly.yaml +++ b/.github/workflows/publish-nightly.yaml @@ -5,19 +5,20 @@ on: push: branches: - master + workflow_dispatch: jobs: publish: runs-on: ubuntu-latest if: ${{ github.repository == 'dotnet/BenchmarkDotNet' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set date run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV - name: Pack run: ./build.cmd pack /p:VersionSuffix=nightly.$DATE.$GITHUB_RUN_NUMBER - name: Upload nupkg to artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: nupkgs path: "**/*.*nupkg" diff --git a/.github/workflows/report-test-results.yaml b/.github/workflows/report-test-results.yaml index 005465b329..f1c627295e 100644 --- a/.github/workflows/report-test-results.yaml +++ b/.github/workflows/report-test-results.yaml @@ -11,13 +11,26 @@ jobs: report: runs-on: ubuntu-latest permissions: write-all + if: ${{ github.event.workflow_run.conclusion != 'cancelled' }} steps: + # Cleanup Old Files + - name: Cleanup Old Files + run: rm -rf $GITHUB_WORKSPACE/*.trx + + # Download the Latest Artifacts with Unique Name - name: Download Artifacts - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v6 with: - workflow: ${{ github.event.workflow_run.workflow_id }} + run_id: ${{ github.event.workflow_run.id }} + + # Display the Structure of Downloaded Files - name: Display structure of downloaded files run: ls -R + + # Display the Contents of .trx Files + - name: Display .trx file contents + run: cat **/*.trx || echo "No .trx files found" + - name: Report tests results uses: AndreyAkinshin/test-reporter@0e2c48ebec2007001dd77dd4bcbcd450b96d5a38 with: diff --git a/.github/workflows/run-tests-selected.yaml b/.github/workflows/run-tests-selected.yaml new file mode 100644 index 0000000000..ddbcc74e09 --- /dev/null +++ b/.github/workflows/run-tests-selected.yaml @@ -0,0 +1,89 @@ +name: run-tests-selected +run-name: Run selected tests (${{ inputs.runs_on }} --framework ${{ inputs.framework}} --filter ${{ inputs.filter }}) + +on: + workflow_dispatch: + inputs: + runs_on: + type: choice + description: GitHub Actions runner image name + required: true + default: ubuntu-latest + options: + - windows-latest + - ubuntu-latest + - macos-latest + - windows-11-arm + - ubuntu-24.04-arm + - macos-13 + project: + type: string + description: Specify test project path + required: true + default: tests/BenchmarkDotNet.IntegrationTests + options: + - tests/BenchmarkDotNet.Tests + - tests/BenchmarkDotNet.IntegrationTests + - tests/BenchmarkDotNet.IntegrationTests.ManualRunning + framework: + type: choice + description: Specify target framework + required: true + options: + - net8.0 + - net462 + filter: + type: string + description: Test filter text (It's used for `dotnet test --filter`) Use default value when running all tests + required: true + default: "BenchmarkDotNet" + iteration_count: + type: number + description: Count of test loop (It's expected to be used for flaky tests) + required: true + default: 1 + +jobs: + test: + name: test (${{ inputs.runs_on }} --framework ${{ inputs.framework}} --filter ${{ inputs.filter }}) + runs-on: ${{ inputs.runs_on }} + timeout-minutes: 60 # Explicitly set timeout. When wrong input parameter is passed. It may continue to run until it times out (Default:360 minutes)) + steps: + - uses: actions/checkout@v4 + + # Setup + - name: Setup + run: | + mkdir artifacts + + # Build + - name: Run build + working-directory: ${{ github.event.inputs.project }} + run: | + dotnet build -c Release --framework ${{ inputs.framework }} -tl:off + + # Test + - name: Run tests + shell: pwsh + working-directory: ${{ github.event.inputs.project }} + run: | + $PSNativeCommandUseErrorActionPreference = $true + $iterationCount = ${{ inputs.iteration_count }} + + foreach($i in 1..$iterationCount) { + Write-Output ('##[group]Executing Iteration: {0}/${{ inputs.iteration_count }}' -f $i) + + dotnet test -c Release --framework ${{ inputs.framework }} --filter ${{ inputs.filter }} -tl:off --no-build --logger "console;verbosity=normal" + + Write-Output '##[endgroup]' + } + + # Upload artifact files that are located at `$(GITHUB_WORKSPACE)/artifacts` directory + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: results + if-no-files-found: ignore + path: | + artifacts/**/* diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 90887cbecd..9d3f25a9e8 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -17,22 +17,20 @@ jobs: - name: Disable Windows Defender run: Set-MpPreference -DisableRealtimeMonitoring $true shell: powershell - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + # Build and Test - name: Run task 'build' shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" - ./build.cmd build + run: ./build.cmd build - name: Run task 'in-tests-core' shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" - ./build.cmd in-tests-core -e + run: ./build.cmd in-tests-core -e + # Upload Artifacts with Unique Name - name: Upload test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: - name: test-windows-core-trx + name: test-windows-core-trx-${{ github.run_id }} path: "**/*.trx" test-windows-full: @@ -41,75 +39,118 @@ jobs: - name: Disable Windows Defender run: Set-MpPreference -DisableRealtimeMonitoring $true shell: powershell - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + # Build and Test - name: Run task 'build' shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" - ./build.cmd build + run: ./build.cmd build - name: Run task 'in-tests-full' shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" - ./build.cmd in-tests-full -e + run: ./build.cmd in-tests-full -e + # Upload Artifacts with Unique Name - name: Upload test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: - name: test-windows-full-trx + name: test-windows-full-trx-${{ github.run_id }} path: "**/*.trx" test-linux: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + # Set up the environment - name: Set up Clang - uses: egor-tensin/setup-clang@v1 + uses: egor-tensin/setup-clang@ef434b41eb33a70396fb336b1bae39c76d740c3d # v1.4 with: version: latest platform: x64 - name: Set up zlib-static run: sudo apt-get install -y libkrb5-dev + - name: Set up node + uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Set up v8 + run: npm install jsvu -g && jsvu --os=linux64 --engines=v8 && echo "$HOME/.jsvu/bin" >> $GITHUB_PATH + - name: Install wasm-tools workload + run: ./build.cmd install-wasm-tools + # Build and Test - name: Run task 'build' run: ./build.cmd build - name: Run task 'unit-tests' run: ./build.cmd unit-tests -e - name: Run task 'in-tests-core' run: ./build.cmd in-tests-core -e + # Upload Artifacts with Unique Name - name: Upload test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: - name: test-linux-trx + name: test-linux-trx-${{ github.run_id }} path: "**/*.trx" test-macos: - runs-on: macos-13 + name: test-macos (${{ matrix.os.arch }}) + runs-on: ${{ matrix.os.runs-on }} + strategy: + matrix: + os: + - runs-on: 'macos-latest' + jsvu-os: 'mac64arm' + arch: 'arm64' + - runs-on: 'macos-13' + jsvu-os: 'mac64' + arch: 'x64' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - name: Set up node + uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Set up v8 + run: npm install jsvu -g && jsvu --os=${{ matrix.os.jsvu-os }} --engines=v8 && echo "$HOME/.jsvu/bin" >> $GITHUB_PATH + - name: Install wasm-tools workload + run: ./build.cmd install-wasm-tools + # Build and Test - name: Run task 'build' run: ./build.cmd build - name: Run task 'unit-tests' run: ./build.cmd unit-tests -e - name: Run task 'in-tests-core' run: ./build.cmd in-tests-core -e + # Upload Artifacts with Unique Name - name: Upload test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: - name: test-macos-trx + name: test-macos(${{ matrix.os.arch }})-trx-${{ github.run_id }} path: "**/*.trx" - + + test-pack: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Clang + uses: egor-tensin/setup-clang@v1 + with: + version: latest + platform: x64 + - name: Set up zlib-static + run: sudo apt-get install -y libkrb5-dev + - name: Run task 'pack' + run: ./build.cmd pack + spellcheck-docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 name: Setup node with: - node-version: "16" + node-version: "22" - name: Install cSpell - run: npm install -g cspell + run: npm install -g cspell@9.0.2 - name: Copy cSpell config run: cp ./build/cSpell.json ./cSpell.json - name: Run cSpell diff --git a/BenchmarkDotNet.sln b/BenchmarkDotNet.sln index ba31552c8a..1df6c0aabd 100644 --- a/BenchmarkDotNet.sln +++ b/BenchmarkDotNet.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.27130.2027 +# Visual Studio Version 17 +VisualStudioVersion = 17.8.34004.107 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{D6597E3A-6892-4A68-8E14-042FC941FDA2}" EndProject @@ -51,6 +51,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BenchmarkDotNet.Diagnostics EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks", "tests\BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks\BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj", "{AACA2C63-A85B-47AB-99FC-72C3FF408B14}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.TestAdapter", "src\BenchmarkDotNet.TestAdapter\BenchmarkDotNet.TestAdapter.csproj", "{4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Diagnostics.dotMemory", "src\BenchmarkDotNet.Diagnostics.dotMemory\BenchmarkDotNet.Diagnostics.dotMemory.csproj", "{2E2283A3-6DA6-4482-8518-99D6D9F689AB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting", "src\BenchmarkDotNet.Exporters.Plotting\BenchmarkDotNet.Exporters.Plotting.csproj", "{B92ECCEF-7C27-4012-9E19-679F3C40A6A6}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting.Tests", "tests\BenchmarkDotNet.Exporters.Plotting.Tests\BenchmarkDotNet.Exporters.Plotting.Tests.csproj", "{199AC83E-30BD-40CD-87CE-0C838AC0320D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -137,6 +145,22 @@ Global {AACA2C63-A85B-47AB-99FC-72C3FF408B14}.Debug|Any CPU.Build.0 = Debug|Any CPU {AACA2C63-A85B-47AB-99FC-72C3FF408B14}.Release|Any CPU.ActiveCfg = Release|Any CPU {AACA2C63-A85B-47AB-99FC-72C3FF408B14}.Release|Any CPU.Build.0 = Release|Any CPU + {4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC}.Release|Any CPU.Build.0 = Release|Any CPU + {2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Release|Any CPU.Build.0 = Release|Any CPU + {B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Release|Any CPU.Build.0 = Release|Any CPU + {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -162,6 +186,10 @@ Global {B620D10A-CD8E-4A34-8B27-FD6257E63AD0} = {63B94FD6-3F3D-4E04-9727-48E86AC4384C} {C5BDA61F-3A56-4B59-901D-0A17E78F4076} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} {AACA2C63-A85B-47AB-99FC-72C3FF408B14} = {14195214-591A-45B7-851A-19D3BA2413F9} + {4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} + {2E2283A3-6DA6-4482-8518-99D6D9F689AB} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} + {B92ECCEF-7C27-4012-9E19-679F3C40A6A6} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} + {199AC83E-30BD-40CD-87CE-0C838AC0320D} = {14195214-591A-45B7-851A-19D3BA2413F9} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {4D9AF12B-1F7F-45A7-9E8C-E4E46ADCBD1F} diff --git a/BenchmarkDotNet.sln.DotSettings b/BenchmarkDotNet.sln.DotSettings index 9542bd9d5b..53f5cf0a60 100644 --- a/BenchmarkDotNet.sln.DotSettings +++ b/BenchmarkDotNet.sln.DotSettings @@ -52,6 +52,8 @@ OSX RT <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> + <Policy><Descriptor Staticness="Instance" AccessRightKinds="Private" Description="Instance fields (private)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> + <Policy><Descriptor Staticness="Static" AccessRightKinds="Private" Description="Static fields (private)"><ElementKinds><Kind Name="FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> True True 983040 @@ -68,6 +70,7 @@ True <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> + True <data><IncludeFilters /><ExcludeFilters><Filter ModuleMask="*" ModuleVersionMask="*" ClassMask="JetBrains.Annotations.*" FunctionMask="*" IsEnabled="True" /><Filter ModuleMask="*" ModuleVersionMask="*" ClassMask="SimpleJson.*" FunctionMask="*" IsEnabled="True" /></ExcludeFilters></data> True True @@ -162,6 +165,8 @@ True True True + True + True True True True @@ -173,6 +178,7 @@ True True True + True True True True diff --git a/LICENSE.md b/LICENSE.md index d20c387e81..6272f97306 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ ### The MIT License -Copyright (c) 2013–2023 .NET Foundation and contributors +Copyright (c) 2013–2025 .NET Foundation and contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/NuGet.Config b/NuGet.Config index 855812b5a5..7507704b8b 100644 --- a/NuGet.Config +++ b/NuGet.Config @@ -13,5 +13,6 @@ + diff --git a/README.md b/README.md index 1d183aeb6d..a9b8a76e9b 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ It's no harder than writing unit tests! Under the hood, it performs a lot of [magic](#automation) that guarantees [reliable and precise](#reliability) results thanks to the [perfolizer](https://github.com/AndreyAkinshin/perfolizer) statistical engine. BenchmarkDotNet protects you from popular benchmarking mistakes and warns you if something is wrong with your benchmark design or obtained measurements. The results are presented in a [user-friendly](#friendliness) form that highlights all the important facts about your experiment. -BenchmarkDotNet is already adopted by [18100+ GitHub projects](https://github.com/dotnet/BenchmarkDotNet/network/dependents) including +BenchmarkDotNet is already adopted by [27000+ GitHub projects](https://github.com/dotnet/BenchmarkDotNet/network/dependents) including [.NET Runtime](https://github.com/dotnet/runtime), [.NET Compiler](https://github.com/dotnet/roslyn), [.NET Performance](https://github.com/dotnet/performance), @@ -125,7 +125,7 @@ Four aspects define the design of these features: ### Simplicity -You shouldn't be an experienced performance engineer if you want to write benchmarks. +You shouldn't have to be an experienced performance engineer if you want to write benchmarks. You can design very complicated performance experiments in the declarative style using simple APIs. For example, if you want to [parameterize](https://benchmarkdotnet.org/articles/features/parameterization.html) your benchmark, @@ -135,8 +135,8 @@ If you want to compare benchmarks with each other, mark one of the benchmarks as the [baseline](https://benchmarkdotnet.org/articles/features/baselines.html) via `[Benchmark(Baseline = true)]`: BenchmarkDotNet will compare it with all of the other benchmarks. If you want to compare performance in different environments, use [jobs](https://benchmarkdotnet.org/articles/configs/jobs.html). -For example, you can run all the benchmarks on .NET Core 3.0 and Mono via - `[SimpleJob(RuntimeMoniker.NetCoreApp30)]` and `[SimpleJob(RuntimeMoniker.Mono)]`. +For example, you can run all the benchmarks on .NET 8.0 and Mono via + `[SimpleJob(RuntimeMoniker.Net80)]` and `[SimpleJob(RuntimeMoniker.Mono)]`. If you don't like attributes, you can call most of the APIs via the fluent style and write code like this: diff --git a/build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj b/build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj index 6b8ecc7ffc..c7b97bc90a 100644 --- a/build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj +++ b/build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj @@ -1,15 +1,15 @@ Exe - net7.0 + net8.0 $(MSBuildProjectDirectory) enable - - - - - + + + + + - \ No newline at end of file + diff --git a/build/BenchmarkDotNet.Build/BuildContext.cs b/build/BenchmarkDotNet.Build/BuildContext.cs index a13c77c5ed..3e3c42c27d 100644 --- a/build/BenchmarkDotNet.Build/BuildContext.cs +++ b/build/BenchmarkDotNet.Build/BuildContext.cs @@ -59,6 +59,9 @@ public BuildContext(ICakeContext context) BuildDirectory = RootDirectory.Combine("build"); ArtifactsDirectory = RootDirectory.Combine("artifacts"); + var toolFileName = context.IsRunningOnWindows() ? "dotnet.exe" : "dotnet"; + var toolFilePath = RootDirectory.Combine(".dotnet").CombineWithFilePath(toolFileName); + context.Tools.RegisterFile(toolFilePath); SolutionFile = RootDirectory.CombineWithFilePath("BenchmarkDotNet.sln"); @@ -154,6 +157,7 @@ public void GenerateFile(FilePath filePath, StringBuilder content) public void GenerateFile(FilePath filePath, string content, bool reportNoChanges = false) { + this.EnsureDirectoryExists(filePath.GetDirectory()); var relativePath = RootDirectory.GetRelativePath(filePath); if (this.FileExists(filePath)) { diff --git a/build/BenchmarkDotNet.Build/Folder.DotSettings b/build/BenchmarkDotNet.Build/Folder.DotSettings index 53109cf04e..539b6fe39e 100644 --- a/build/BenchmarkDotNet.Build/Folder.DotSettings +++ b/build/BenchmarkDotNet.Build/Folder.DotSettings @@ -1,4 +1,7 @@  <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> + <Policy><Descriptor Staticness="Instance" AccessRightKinds="Private" Description="Instance fields (private)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> + <Policy><Descriptor Staticness="Static" AccessRightKinds="Private" Description="Static fields (private)"><ElementKinds><Kind Name="FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> + True True \ No newline at end of file diff --git a/build/BenchmarkDotNet.Build/Meta/Repo.cs b/build/BenchmarkDotNet.Build/Meta/Repo.cs index 38215fcce8..22a02d51bb 100644 --- a/build/BenchmarkDotNet.Build/Meta/Repo.cs +++ b/build/BenchmarkDotNet.Build/Meta/Repo.cs @@ -12,7 +12,7 @@ public static class Repo public const string HttpsUrlBase = $"https://github.com/{Owner}/{Name}"; public const string HttpsGitUrl = $"{HttpsUrlBase}.git"; - public const string ChangelogDetailsBranch = "docs-changelog-details"; + public const string ChangelogBranch = "docs-changelog"; public const string DocsStableBranch = "docs-stable"; public const string MasterBranch = "master"; diff --git a/build/BenchmarkDotNet.Build/Options/KnownOptions.cs b/build/BenchmarkDotNet.Build/Options/KnownOptions.cs index c5101e1d52..6da4e390ed 100644 --- a/build/BenchmarkDotNet.Build/Options/KnownOptions.cs +++ b/build/BenchmarkDotNet.Build/Options/KnownOptions.cs @@ -29,6 +29,12 @@ public static class KnownOptions Aliases = new[] { "-d" } }; + public static readonly BoolOption ForceClone = new("--force-clone") + { + Description = "Forces re-cloning of the changelog repository, deleting any existing directory.", + Aliases = new[] { "-f" } + }; + public static readonly BoolOption Help = new("--help") { Description = "Prints help information", diff --git a/build/BenchmarkDotNet.Build/Program.cs b/build/BenchmarkDotNet.Build/Program.cs index f099c38094..5fd25cc298 100644 --- a/build/BenchmarkDotNet.Build/Program.cs +++ b/build/BenchmarkDotNet.Build/Program.cs @@ -28,10 +28,10 @@ public HelpInfo GetHelp() { return new HelpInfo { - Examples = new[] - { + Examples = + [ new Example(Name) - } + ] }; } } @@ -48,10 +48,30 @@ public HelpInfo GetHelp() { return new HelpInfo { - Examples = new[] - { + Examples = + [ new Example(Name).WithMsBuildArgument("Configuration", "Debug") - } + ] + }; + } +} + +[TaskName(Name)] +[TaskDescription("Install wasm-tools workload")] +public class InstallWasmToolsWorkload : FrostingTask, IHelpProvider +{ + private const string Name = "install-wasm-tools"; + + public override void Run(BuildContext context) => context.BuildRunner.InstallWorkload("wasm-tools"); + + public HelpInfo GetHelp() + { + return new HelpInfo + { + Examples = + [ + new Example(Name) + ] }; } } @@ -68,12 +88,12 @@ public HelpInfo GetHelp() { return new HelpInfo { - Examples = new[] - { + Examples = + [ new Example(Name) .WithArgument(KnownOptions.Exclusive) .WithArgument(KnownOptions.Verbosity, "Diagnostic") - } + ] }; } } @@ -93,12 +113,12 @@ public class InTestsFullTask : FrostingTask, IHelpProvider } [TaskName(Name)] -[TaskDescription("Run integration tests using .NET 7 (slow)")] +[TaskDescription("Run integration tests using .NET 8 (slow)")] [IsDependentOn(typeof(BuildTask))] public class InTestsCoreTask : FrostingTask, IHelpProvider { private const string Name = "in-tests-core"; - public override void Run(BuildContext context) => context.UnitTestRunner.RunInTests("net7.0"); + public override void Run(BuildContext context) => context.UnitTestRunner.RunInTests("net8.0"); public HelpInfo GetHelp() => new(); } @@ -125,68 +145,69 @@ public HelpInfo GetHelp() { return new HelpInfo { - Examples = new[] - { + Examples = + [ new Example(Name) .WithMsBuildArgument("VersionPrefix", "0.1.1729") .WithMsBuildArgument("VersionSuffix", "preview"), new Example(Name).WithArgument(KnownOptions.Stable) - } + ] }; } } [TaskName(Name)] -[TaskDescription("Update generated documentation files")] -public class DocsUpdateTask : FrostingTask, IHelpProvider +[TaskDescription("Fetch changelog files")] +public class DocsFetchTask : FrostingTask, IHelpProvider { - private const string Name = "docs-update"; - public override void Run(BuildContext context) => context.DocumentationRunner.Update(); + private const string Name = "docs-fetch"; + public override void Run(BuildContext context) => context.DocumentationRunner.Fetch(); public HelpInfo GetHelp() { return new HelpInfo { Description = $"This task updates the following files:\n" + - $"* README.md (the number of dependent projects number)\n" + + $"* Clones branch 'docs-changelog' to docs/_changelog\n" + $"* Last changelog footer (if {KnownOptions.Stable.CommandLineName} is specified)\n" + $"* All changelog details in docs/_changelog\n" + - $" (This dir is a cloned version of this repo from branch {Repo.ChangelogDetailsBranch})", - Options = new IOption[] { KnownOptions.DocsPreview, KnownOptions.DocsDepth }, - EnvironmentVariables = new[] { EnvVar.GitHubToken }, - Examples = new[] - { + $" (This dir is a cloned version of this repo from branch {Repo.ChangelogBranch})", + Options = [KnownOptions.DocsPreview, KnownOptions.DocsDepth, KnownOptions.ForceClone], + EnvironmentVariables = [EnvVar.GitHubToken], + Examples = + [ new Example(Name) .WithArgument(KnownOptions.DocsDepth, "3") .WithArgument(KnownOptions.DocsPreview) - } + .WithArgument(KnownOptions.ForceClone) + ] }; } } [TaskName(Name)] -[TaskDescription("Prepare auxiliary documentation files")] -public class DocsPrepareTask : FrostingTask, IHelpProvider +[TaskDescription("Generate auxiliary documentation files")] +public class DocsGenerateTask : FrostingTask, IHelpProvider { - private const string Name = "docs-prepare"; - public override void Run(BuildContext context) => context.DocumentationRunner.Prepare(); + private const string Name = "docs-generate"; + public override void Run(BuildContext context) => context.DocumentationRunner.Generate(); public HelpInfo GetHelp() { return new HelpInfo { - Options = new IOption[] { KnownOptions.DocsPreview }, - Examples = new[] - { + Options = [KnownOptions.DocsPreview], + Examples = + [ new Example(Name).WithArgument(KnownOptions.DocsPreview) - } + ] }; } } [TaskName(Name)] [TaskDescription("Build final documentation")] -[IsDependentOn(typeof(DocsPrepareTask))] +[IsDependentOn(typeof(DocsGenerateTask))] public class DocsBuildTask : FrostingTask, IHelpProvider { private const string Name = "docs-build"; @@ -195,11 +216,11 @@ public class DocsBuildTask : FrostingTask, IHelpProvider public HelpInfo GetHelp() => new() { Description = "The 'build' task should be run manually to build api docs", - Options = new IOption[] { KnownOptions.DocsPreview }, - Examples = new[] - { + Options = [KnownOptions.DocsPreview], + Examples = + [ new Example(Name).WithArgument(KnownOptions.DocsPreview) - } + ] }; } @@ -207,7 +228,8 @@ public class DocsBuildTask : FrostingTask, IHelpProvider [TaskDescription("Release new version")] [IsDependentOn(typeof(BuildTask))] [IsDependentOn(typeof(PackTask))] -[IsDependentOn(typeof(DocsUpdateTask))] +[IsDependentOn(typeof(DocsFetchTask))] +[IsDependentOn(typeof(DocsGenerateTask))] [IsDependentOn(typeof(DocsBuildTask))] public class ReleaseTask : FrostingTask, IHelpProvider { @@ -216,10 +238,10 @@ public class ReleaseTask : FrostingTask, IHelpProvider public HelpInfo GetHelp() => new() { - Options = new IOption[] { KnownOptions.NextVersion, KnownOptions.Push }, - EnvironmentVariables = new[] { EnvVar.GitHubToken, EnvVar.NuGetToken }, - Examples = new[] - { + Options = [KnownOptions.NextVersion, KnownOptions.Push], + EnvironmentVariables = [EnvVar.GitHubToken, EnvVar.NuGetToken], + Examples = + [ new Example(Name) .WithArgument(KnownOptions.Stable) .WithArgument(KnownOptions.NextVersion, "0.1.1729") @@ -227,6 +249,6 @@ public class ReleaseTask : FrostingTask, IHelpProvider new Example(Name) .WithArgument(KnownOptions.Stable) .WithArgument(KnownOptions.Push) - } + ] }; } \ No newline at end of file diff --git a/build/BenchmarkDotNet.Build/Runners/BuildRunner.cs b/build/BenchmarkDotNet.Build/Runners/BuildRunner.cs index 01c490fce3..a38ce7e79d 100644 --- a/build/BenchmarkDotNet.Build/Runners/BuildRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/BuildRunner.cs @@ -5,6 +5,7 @@ using Cake.Common.Tools.DotNet.Build; using Cake.Common.Tools.DotNet.Pack; using Cake.Common.Tools.DotNet.Restore; +using Cake.Common.Tools.DotNet.Workload.Install; using Cake.Core; using Cake.Core.IO; @@ -28,6 +29,16 @@ public void Restore() }); } + public void InstallWorkload(string workloadId) + { + context.DotNetWorkloadInstall(workloadId, + new DotNetWorkloadInstallSettings + { + IncludePreviews = true, + NoCache = true + }); + } + public void Build() { context.Information("BuildSystemProvider: " + context.BuildSystem().Provider); diff --git a/build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogBuilder.cs b/build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogBuilder.cs new file mode 100644 index 0000000000..5bfc8c91b2 --- /dev/null +++ b/build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogBuilder.cs @@ -0,0 +1,256 @@ +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using BenchmarkDotNet.Build.Meta; +using BenchmarkDotNet.Build.Options; +using Cake.Common.Diagnostics; +using Cake.Common.IO; +using Cake.Core.IO; +using Cake.FileHelpers; + +namespace BenchmarkDotNet.Build.Runners.Changelog; + +public class ChangelogBuilder +{ + private readonly BuildContext context; + private readonly bool preview; + private readonly string depth; + private readonly bool forceClone; + + /// + /// Directory with original changelog part files from branch 'docs-changelog' + /// + public DirectoryPath SrcDirectory { get; } + + /// + /// Final changelog files to be used by docfx + /// + public DirectoryPath DocfxDirectory { get; } + + public ChangelogBuilder(BuildContext context) + { + this.context = context; + preview = KnownOptions.DocsPreview.Resolve(context); + depth = KnownOptions.DocsDepth.Resolve(context); + forceClone = KnownOptions.ForceClone.Resolve(context); + + var docsDirectory = context.RootDirectory.Combine("docs"); + SrcDirectory = docsDirectory.Combine("_changelog"); + DocfxDirectory = docsDirectory.Combine("changelog"); + } + + public void Fetch() + { + EnvVar.GitHubToken.AssertHasValue(); + + EnsureSrcDirectoryExist(forceClone); + + var history = context.VersionHistory; + var stableVersionCount = history.StableVersions.Length; + + if (depth.Equals("all", StringComparison.OrdinalIgnoreCase)) + { + FetchDetails( + history.StableVersions.First(), + history.FirstCommit); + + for (var i = 1; i < stableVersionCount; i++) + FetchDetails( + history.StableVersions[i], + history.StableVersions[i - 1]); + } + else if (depth != "") + { + if (!int.TryParse(depth, CultureInfo.InvariantCulture, out var depthValue)) + throw new InvalidDataException($"Failed to parse the depth value: '{depth}'"); + + for (var i = Math.Max(stableVersionCount - depthValue, 1); i < stableVersionCount; i++) + FetchDetails( + history.StableVersions[i], + history.StableVersions[i - 1]); + } + + if (preview) + FetchDetails( + history.CurrentVersion, + history.StableVersions.Last(), + "HEAD"); + } + + private void FetchDetails(string version, string versionPrevious, string lastCommit = "") + { + EnsureSrcDirectoryExist(); + context.Information($"Downloading changelog details for v{version}"); + var detailsDirectory = SrcDirectory.Combine("details"); + ChangelogDetailsBuilder.Run(context, detailsDirectory, version, versionPrevious, lastCommit); + } + + public void Generate() + { + GenerateLastFooter(); + + foreach (var version in context.VersionHistory.StableVersions) + GenerateVersion(version); + if (preview) + GenerateVersion(context.VersionHistory.CurrentVersion); + + GenerateIndex(); + GenerateFull(); + GenerateToc(); + } + + public void GenerateLastFooter() + { + var version = context.VersionHistory.CurrentVersion; + var previousVersion = context.VersionHistory.StableVersions.Last(); + var date = KnownOptions.Stable.Resolve(context) + ? DateTime.Now.ToString("MMMM dd, yyyy", CultureInfo.InvariantCulture) + : "TBA"; + + var content = new StringBuilder(); + content.AppendLine($"_Date: {date}_"); + content.AppendLine(""); + content.AppendLine( + $"_Milestone: [v{version}](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av{version})_"); + content.AppendLine( + $"([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v{previousVersion}...v{version}))"); + content.AppendLine(""); + content.AppendLine("_NuGet Packages:_"); + foreach (var packageName in context.NuGetPackageNames) + content.AppendLine($"* https://www.nuget.org/packages/{packageName}/{version}"); + + var fileName = "v" + context.VersionHistory.CurrentVersion + ".md"; + var filePath = SrcDirectory.Combine("footer").CombineWithFilePath(fileName); + context.GenerateFile(filePath, content); + } + + private void GenerateVersion(string version) + { + EnsureSrcDirectoryExist(); + var md = $"v{version}.md"; + var header = SrcDirectory.Combine("header").CombineWithFilePath(md); + var footer = SrcDirectory.Combine("footer").CombineWithFilePath(md); + var details = SrcDirectory.Combine("details").CombineWithFilePath(md); + var release = DocfxDirectory.CombineWithFilePath(md); + + var content = new StringBuilder(); + content.AppendLine("---"); + content.AppendLine("uid: changelog.v" + version); + content.AppendLine("---"); + content.AppendLine(""); + content.AppendLine("# BenchmarkDotNet v" + version); + content.AppendLine(""); + content.AppendLine(""); + + if (context.FileExists(header)) + { + content.AppendLine(context.FileReadText(header)); + content.AppendLine(""); + content.AppendLine(""); + } + + if (context.FileExists(details)) + { + content.AppendLine(context.FileReadText(details)); + content.AppendLine(""); + content.AppendLine(""); + } + + if (context.FileExists(footer)) + { + content.AppendLine("## Additional details"); + content.AppendLine(""); + content.AppendLine(context.FileReadText(footer)); + } + + context.GenerateFile(release, content.ToString()); + } + + private void GenerateIndex() + { + var content = new StringBuilder(); + content.AppendLine("---"); + content.AppendLine("uid: changelog"); + content.AppendLine("---"); + content.AppendLine(""); + content.AppendLine("# ChangeLog"); + content.AppendLine(""); + if (preview) + content.AppendLine($"* @changelog.v{context.VersionHistory.CurrentVersion}"); + foreach (var version in context.VersionHistory.StableVersions.Reverse()) + content.AppendLine($"* @changelog.v{version}"); + content.AppendLine("* @changelog.full"); + + context.GenerateFile(DocfxDirectory.CombineWithFilePath("index.md"), content); + } + + private void GenerateFull() + { + var content = new StringBuilder(); + content.AppendLine("---"); + content.AppendLine("uid: changelog.full"); + content.AppendLine("---"); + content.AppendLine(""); + content.AppendLine("# Full ChangeLog"); + content.AppendLine(""); + if (preview) + content.AppendLine( + $"[!include[v{context.VersionHistory.CurrentVersion}](v{context.VersionHistory.CurrentVersion}.md)]"); + foreach (var version in context.VersionHistory.StableVersions.Reverse()) + content.AppendLine($"[!include[v{version}](v{version}.md)]"); + + context.GenerateFile(DocfxDirectory.CombineWithFilePath("full.md"), content); + } + + private void GenerateToc() + { + var content = new StringBuilder(); + + if (preview) + { + content.AppendLine($"- name: v{context.VersionHistory.CurrentVersion}"); + content.AppendLine($" href: v{context.VersionHistory.CurrentVersion}.md"); + } + + foreach (var version in context.VersionHistory.StableVersions.Reverse()) + { + content.AppendLine($"- name: v{version}"); + content.AppendLine($" href: v{version}.md"); + } + + content.AppendLine("- name: Full ChangeLog"); + content.AppendLine(" href: full.md"); + + context.GenerateFile(DocfxDirectory.CombineWithFilePath("toc.yml"), content); + } + + private void EnsureSrcDirectoryExist(bool forceClone = false) + { + void Log(string message) => context.Information($"[Changelog] {message}"); + + Log($"Preparing git sub-repository for changelog branch '{Repo.ChangelogBranch}'. " + + $"Target directory: '{SrcDirectory}'."); + if (context.DirectoryExists(SrcDirectory) && forceClone) + { + Log($"Directory '{SrcDirectory}' already exists and forceClean is specified. " + + $"Deleting the current directory..."); + context.DeleteDirectory( + SrcDirectory, + new DeleteDirectorySettings { Force = true, Recursive = true }); + Log($"Directory '{SrcDirectory}' deleted successfully."); + } + + if (!context.DirectoryExists(SrcDirectory)) + { + Log($"Cloning branch '{Repo.ChangelogBranch}' from '{Repo.HttpsGitUrl}' to '{SrcDirectory}'."); + context.GitRunner.Clone(SrcDirectory, Repo.HttpsGitUrl, Repo.ChangelogBranch); + Log($"Clone completed: '{Repo.ChangelogBranch}' -> '{SrcDirectory}'."); + } + else + { + Log($"Directory '{SrcDirectory}' already exists. Skipping clone."); + } + } +} \ No newline at end of file diff --git a/build/BenchmarkDotNet.Build/ChangeLogBuilder.cs b/build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogDetailsBuilder.cs similarity index 93% rename from build/BenchmarkDotNet.Build/ChangeLogBuilder.cs rename to build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogDetailsBuilder.cs index e7769e1fbd..27540402e3 100644 --- a/build/BenchmarkDotNet.Build/ChangeLogBuilder.cs +++ b/build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogDetailsBuilder.cs @@ -10,15 +10,15 @@ using Cake.Core.IO; using Octokit; -namespace BenchmarkDotNet.Build; +namespace BenchmarkDotNet.Build.Runners.Changelog; -public static class ChangeLogBuilder +public static class ChangelogDetailsBuilder { - private class Config + private class Config(string currentVersion, string previousVersion, string lastCommit) { - public string CurrentVersion { get; } - public string PreviousVersion { get; } - public string LastCommit { get; } + public string CurrentVersion { get; } = currentVersion; + public string PreviousVersion { get; } = previousVersion; + public string LastCommit { get; } = lastCommit; public void Deconstruct(out string currentMilestone, out string previousMilestone, out string lastCommit) { @@ -26,13 +26,6 @@ public void Deconstruct(out string currentMilestone, out string previousMileston previousMilestone = PreviousVersion; lastCommit = LastCommit; } - - public Config(string currentVersion, string previousVersion, string lastCommit) - { - CurrentVersion = currentVersion; - PreviousVersion = previousVersion; - LastCommit = lastCommit; - } } private class MarkdownBuilder diff --git a/build/BenchmarkDotNet.Build/Runners/DocumentationRunner.cs b/build/BenchmarkDotNet.Build/Runners/DocumentationRunner.cs index 2d2128a86f..e1d1af7469 100644 --- a/build/BenchmarkDotNet.Build/Runners/DocumentationRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/DocumentationRunner.cs @@ -1,11 +1,9 @@ -using System; -using System.Globalization; using System.IO; using System.Linq; using System.Text; using BenchmarkDotNet.Build.Helpers; using BenchmarkDotNet.Build.Meta; -using BenchmarkDotNet.Build.Options; +using BenchmarkDotNet.Build.Runners.Changelog; using Cake.Common.Diagnostics; using Cake.Common.IO; using Cake.Core.IO; @@ -16,85 +14,47 @@ namespace BenchmarkDotNet.Build.Runners; public class DocumentationRunner { private readonly BuildContext context; - private readonly bool preview; - private readonly string depth; - - public DirectoryPath ChangelogDirectory { get; } - public DirectoryPath ChangelogSrcDirectory { get; } - private readonly DirectoryPath changelogDetailsDirectory; + private readonly ChangelogBuilder changelogBuilder; private readonly DirectoryPath docsGeneratedDirectory; private readonly FilePath docfxJsonFile; private readonly FilePath redirectFile; private readonly FilePath readmeFile; private readonly FilePath rootIndexFile; - private readonly FilePath changelogIndexFile; - private readonly FilePath changelogFullFile; - private readonly FilePath changelogTocFile; - private readonly FilePath lastFooterFile; + + public DirectoryPath ChangelogSrcDirectory => changelogBuilder.SrcDirectory; public DocumentationRunner(BuildContext context) { this.context = context; - preview = KnownOptions.DocsPreview.Resolve(context); - depth = KnownOptions.DocsDepth.Resolve(context); + changelogBuilder = new ChangelogBuilder(context); var docsDirectory = context.RootDirectory.Combine("docs"); - ChangelogDirectory = docsDirectory.Combine("changelog"); - ChangelogSrcDirectory = docsDirectory.Combine("_changelog"); - changelogDetailsDirectory = ChangelogSrcDirectory.Combine("details"); docsGeneratedDirectory = docsDirectory.Combine("_site"); - redirectFile = docsDirectory.Combine("_redirects").CombineWithFilePath("_redirects"); docfxJsonFile = docsDirectory.CombineWithFilePath("docfx.json"); readmeFile = context.RootDirectory.CombineWithFilePath("README.md"); rootIndexFile = docsDirectory.CombineWithFilePath("index.md"); - changelogIndexFile = ChangelogDirectory.CombineWithFilePath("index.md"); - changelogFullFile = ChangelogDirectory.CombineWithFilePath("full.md"); - changelogTocFile = ChangelogDirectory.CombineWithFilePath("toc.yml"); - lastFooterFile = ChangelogSrcDirectory.Combine("footer") - .CombineWithFilePath("v" + context.VersionHistory.CurrentVersion + ".md"); } - public void Update() + public void Fetch() { EnvVar.GitHubToken.AssertHasValue(); + changelogBuilder.Fetch(); + } - UpdateReadme(); - UpdateLastFooter(); - - EnsureChangelogDetailsExist(); - - var history = context.VersionHistory; - var stableVersionCount = history.StableVersions.Length; - - if (depth.Equals("all", StringComparison.OrdinalIgnoreCase)) - { - DocfxChangelogDownload( - history.StableVersions.First(), - history.FirstCommit); - - for (var i = 1; i < stableVersionCount; i++) - DocfxChangelogDownload( - history.StableVersions[i], - history.StableVersions[i - 1]); - } - else if (depth != "") - { - if (!int.TryParse(depth, CultureInfo.InvariantCulture, out var depthValue)) - throw new InvalidDataException($"Failed to parse the depth value: '{depth}'"); + public void Generate() + { + changelogBuilder.Generate(); - for (var i = Math.Max(stableVersionCount - depthValue, 1); i < stableVersionCount; i++) - DocfxChangelogDownload( - history.StableVersions[i], - history.StableVersions[i - 1]); - } + UpdateReadme(); + GenerateIndexMd(); + } - if (preview) - DocfxChangelogDownload( - history.CurrentVersion, - history.StableVersions.Last(), - "HEAD"); + public void Build() + { + RunDocfx(); + GenerateRedirects(); } private void UpdateReadme() @@ -108,24 +68,6 @@ private void UpdateReadme() context.GenerateFile(context.ReadmeFile, content, true); } - public void Prepare() - { - foreach (var version in context.VersionHistory.StableVersions) - DocfxChangelogGenerate(version); - if (preview) - DocfxChangelogGenerate(context.VersionHistory.CurrentVersion); - - GenerateIndexMd(); - GenerateChangelogIndex(); - GenerateChangelogFull(); - GenerateChangelogToc(); - } - - public void Build() - { - RunDocfx(); - GenerateRedirects(); - } private void RunDocfx() { @@ -149,124 +91,6 @@ private void GenerateIndexMd() context.GenerateFile(rootIndexFile, content); } - private void GenerateChangelogToc() - { - var content = new StringBuilder(); - - if (preview) - { - content.AppendLine($"- name: v{context.VersionHistory.CurrentVersion}"); - content.AppendLine($" href: v{context.VersionHistory.CurrentVersion}.md"); - } - - foreach (var version in context.VersionHistory.StableVersions.Reverse()) - { - content.AppendLine($"- name: v{version}"); - content.AppendLine($" href: v{version}.md"); - } - - content.AppendLine("- name: Full ChangeLog"); - content.AppendLine(" href: full.md"); - - context.GenerateFile(changelogTocFile, content); - } - - private void GenerateChangelogFull() - { - var content = new StringBuilder(); - content.AppendLine("---"); - content.AppendLine("uid: changelog.full"); - content.AppendLine("---"); - content.AppendLine(""); - content.AppendLine("# Full ChangeLog"); - content.AppendLine(""); - if (preview) - content.AppendLine( - $"[!include[v{context.VersionHistory.CurrentVersion}](v{context.VersionHistory.CurrentVersion}.md)]"); - foreach (var version in context.VersionHistory.StableVersions.Reverse()) - content.AppendLine($"[!include[v{version}](v{version}.md)]"); - - context.GenerateFile(changelogFullFile, content); - } - - private void GenerateChangelogIndex() - { - var content = new StringBuilder(); - content.AppendLine("---"); - content.AppendLine("uid: changelog"); - content.AppendLine("---"); - content.AppendLine(""); - content.AppendLine("# ChangeLog"); - content.AppendLine(""); - if (preview) - content.AppendLine($"* @changelog.v{context.VersionHistory.CurrentVersion}"); - foreach (var version in context.VersionHistory.StableVersions.Reverse()) - content.AppendLine($"* @changelog.v{version}"); - content.AppendLine("* @changelog.full"); - - context.GenerateFile(changelogIndexFile, content); - } - - private void DocfxChangelogGenerate(string version) - { - EnsureChangelogDetailsExist(); - var md = $"v{version}.md"; - var header = ChangelogSrcDirectory.Combine("header").CombineWithFilePath(md); - var footer = ChangelogSrcDirectory.Combine("footer").CombineWithFilePath(md); - var details = ChangelogSrcDirectory.Combine("details").CombineWithFilePath(md); - var release = ChangelogDirectory.CombineWithFilePath(md); - - var content = new StringBuilder(); - content.AppendLine("---"); - content.AppendLine("uid: changelog.v" + version); - content.AppendLine("---"); - content.AppendLine(""); - content.AppendLine("# BenchmarkDotNet v" + version); - content.AppendLine(""); - content.AppendLine(""); - - if (context.FileExists(header)) - { - content.AppendLine(context.FileReadText(header)); - content.AppendLine(""); - content.AppendLine(""); - } - - if (context.FileExists(details)) - { - content.AppendLine(context.FileReadText(details)); - content.AppendLine(""); - content.AppendLine(""); - } - - if (context.FileExists(footer)) - { - content.AppendLine("## Additional details"); - content.AppendLine(""); - content.AppendLine(context.FileReadText(footer)); - } - - context.GenerateFile(release, content.ToString()); - } - - private void EnsureChangelogDetailsExist(bool forceClean = false) - { - if (context.DirectoryExists(changelogDetailsDirectory) && forceClean) - context.DeleteDirectory( - changelogDetailsDirectory, - new DeleteDirectorySettings { Force = true, Recursive = true }); - - if (!context.DirectoryExists(changelogDetailsDirectory)) - context.GitRunner.Clone(changelogDetailsDirectory, Repo.HttpsGitUrl, Repo.ChangelogDetailsBranch); - } - - private void DocfxChangelogDownload(string version, string versionPrevious, string lastCommit = "") - { - EnsureChangelogDetailsExist(); - context.Information($"Downloading changelog details for v{version}"); - ChangeLogBuilder.Run(context, changelogDetailsDirectory, version, versionPrevious, lastCommit); - } - private void GenerateRedirects() { if (!context.FileExists(redirectFile)) @@ -300,27 +124,4 @@ private void GenerateRedirects() context.GenerateFile(fullFilePath, content); } } - - private void UpdateLastFooter() - { - var version = context.VersionHistory.CurrentVersion; - var previousVersion = context.VersionHistory.StableVersions.Last(); - var date = KnownOptions.Stable.Resolve(context) - ? DateTime.Now.ToString("MMMM dd, yyyy", CultureInfo.InvariantCulture) - : "TBA"; - - var content = new StringBuilder(); - content.AppendLine($"_Date: {date}_"); - content.AppendLine(""); - content.AppendLine( - $"_Milestone: [v{version}](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av{version})_"); - content.AppendLine( - $"([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v{previousVersion}...v{version}))"); - content.AppendLine(""); - content.AppendLine("_NuGet Packages:_"); - foreach (var packageName in context.NuGetPackageNames) - content.AppendLine($"* https://www.nuget.org/packages/{packageName}/{version}"); - - context.GenerateFile(lastFooterFile, content); - } } \ No newline at end of file diff --git a/build/BenchmarkDotNet.Build/Runners/ReleaseRunner.cs b/build/BenchmarkDotNet.Build/Runners/ReleaseRunner.cs index fb9aa895ec..e9d1fcb9de 100644 --- a/build/BenchmarkDotNet.Build/Runners/ReleaseRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/ReleaseRunner.cs @@ -143,6 +143,7 @@ private void PublishGitHubRelease() Draft = false, Prerelease = false, GenerateReleaseNotes = false, + DiscussionCategoryName = "Announcements", Body = notes }).Wait(); context.Information(" Success"); diff --git a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs index 48d5183e14..89b777d09f 100644 --- a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs @@ -4,31 +4,29 @@ using Cake.Common.Tools.DotNet; using Cake.Common.Tools.DotNet.Test; using Cake.Core.IO; +using System.Runtime.InteropServices; namespace BenchmarkDotNet.Build.Runners; -public class UnitTestRunner +public class UnitTestRunner(BuildContext context) { - private readonly BuildContext context; + private FilePath UnitTestsProjectFile { get; } = context.RootDirectory + .Combine("tests") + .Combine("BenchmarkDotNet.Tests") + .CombineWithFilePath("BenchmarkDotNet.Tests.csproj"); - private FilePath UnitTestsProjectFile { get; } - private FilePath IntegrationTestsProjectFile { get; } - private DirectoryPath TestOutputDirectory { get; } + private FilePath ExporterTestsProjectFile { get; } = context.RootDirectory + .Combine("tests") + .Combine("BenchmarkDotNet.Exporters.Plotting.Tests") + .CombineWithFilePath("BenchmarkDotNet.Exporters.Plotting.Tests.csproj"); - public UnitTestRunner(BuildContext context) - { - this.context = context; - UnitTestsProjectFile = context.RootDirectory - .Combine("tests") - .Combine("BenchmarkDotNet.Tests") - .CombineWithFilePath("BenchmarkDotNet.Tests.csproj"); - IntegrationTestsProjectFile = context.RootDirectory - .Combine("tests") - .Combine("BenchmarkDotNet.IntegrationTests") - .CombineWithFilePath("BenchmarkDotNet.IntegrationTests.csproj"); - TestOutputDirectory = context.RootDirectory - .Combine("TestResults"); - } + private FilePath IntegrationTestsProjectFile { get; } = context.RootDirectory + .Combine("tests") + .Combine("BenchmarkDotNet.IntegrationTests") + .CombineWithFilePath("BenchmarkDotNet.IntegrationTests.csproj"); + + private DirectoryPath TestOutputDirectory { get; } = context.RootDirectory + .Combine("TestResults"); private DotNetTestSettings GetTestSettingsParameters(FilePath logFile, string tfm) { @@ -50,7 +48,8 @@ private DotNetTestSettings GetTestSettingsParameters(FilePath logFile, string tf private void RunTests(FilePath projectFile, string alias, string tfm) { var os = Utils.GetOs(); - var trxFileName = $"{os}-{alias}-{tfm}.trx"; + var arch = RuntimeInformation.OSArchitecture.ToString().ToLower(); + var trxFileName = $"{os}({arch})-{alias}-{tfm}.trx"; var trxFile = TestOutputDirectory.CombineWithFilePath(trxFileName); var settings = GetTestSettingsParameters(trxFile, tfm); @@ -58,14 +57,15 @@ private void RunTests(FilePath projectFile, string alias, string tfm) context.DotNetTest(projectFile.FullPath, settings); } - private void RunUnitTests(string tfm) => RunTests(UnitTestsProjectFile, "unit", tfm); + private void RunUnitTests(string tfm) + { + RunTests(UnitTestsProjectFile, "unit", tfm); + RunTests(ExporterTestsProjectFile, "exporters", tfm); + } public void RunUnitTests() { - var targetFrameworks = context.IsRunningOnWindows() - ? new[] { "net462", "net7.0" } - : new[] { "net7.0" }; - + string[] targetFrameworks = context.IsRunningOnWindows() ? ["net462", "net8.0"] : ["net8.0"]; foreach (var targetFramework in targetFrameworks) RunUnitTests(targetFramework); } diff --git a/build/build.ps1 b/build/build.ps1 index 2f6d5a5d11..d4db557409 100755 --- a/build/build.ps1 +++ b/build/build.ps1 @@ -58,11 +58,10 @@ if (!(Test-Path $InstallPath)) { $ScriptPath = Join-Path $InstallPath 'dotnet-install.ps1' (New-Object System.Net.WebClient).DownloadFile($DotNetInstallerUri, $ScriptPath); & $ScriptPath -JSonFile $GlobalJsonPath -InstallDir $InstallPath; - - Remove-PathVariable "$InstallPath" - $env:PATH = "$InstallPath;$env:PATH" } +Remove-PathVariable "$InstallPath" +$env:PATH = "$InstallPath;$env:PATH" $env:DOTNET_ROOT=$InstallPath ########################################################################### diff --git a/build/build.sh b/build/build.sh index ebf8ef04bd..e07aecf5ff 100755 --- a/build/build.sh +++ b/build/build.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Define variables -SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd ) +PROJECT_ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd ) ########################################################################### # INSTALL .NET CORE CLI @@ -12,17 +12,17 @@ export DOTNET_CLI_TELEMETRY_OPTOUT=1 export DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER=0 export DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX=2 -if [ ! -d "$SCRIPT_DIR/.dotnet" ]; then - mkdir "$SCRIPT_DIR/.dotnet" - curl -Lsfo "$SCRIPT_DIR/.dotnet/dotnet-install.sh" https://dot.net/v1/dotnet-install.sh - bash "$SCRIPT_DIR/.dotnet/dotnet-install.sh" --jsonfile ./build/sdk/global.json --install-dir .dotnet --no-path +if [ ! -d "$PROJECT_ROOT/.dotnet" ]; then + mkdir "$PROJECT_ROOT/.dotnet" + curl -Lsfo "$PROJECT_ROOT/.dotnet/dotnet-install.sh" https://dot.net/v1/dotnet-install.sh + bash "$PROJECT_ROOT/.dotnet/dotnet-install.sh" --jsonfile ./build/sdk/global.json --install-dir .dotnet --no-path fi -export PATH="$SCRIPT_DIR/.dotnet":$PATH -export DOTNET_ROOT="$SCRIPT_DIR/.dotnet" +export PATH="$PROJECT_ROOT/.dotnet":$PATH +export DOTNET_ROOT="$PROJECT_ROOT/.dotnet" ########################################################################### # RUN BUILD SCRIPT ########################################################################### -dotnet run --configuration Release --project ./build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj -- "$@" +dotnet run --configuration Release --project "$PROJECT_ROOT/build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj" -- "$@" diff --git a/build/cSpell.json b/build/cSpell.json index a224a62a9c..a34d415fe3 100644 --- a/build/cSpell.json +++ b/build/cSpell.json @@ -12,6 +12,7 @@ "Cygwin", "Diagnoser", "diagnosers", + "diagsession", "disassemblers", "disassm", "Jits", @@ -29,6 +30,8 @@ "Pseudocode", "runtimes", "Serilog", + "vsprofiler", + "vstest", "Tailcall", "toolchains", "unmanaged" @@ -36,8 +39,10 @@ "ignoreWords": [ "Akinshin", "Andrey", + "Cassell", "Expecto", "Jint", + "JITted", "LoongArch64", "macrobenchmark", "MediatR", @@ -46,6 +51,9 @@ "NodaTime", "Npgsql", "Sitnik's", + "Sitnik", + "Stepanov", + "Yegor", "Wojciech", "Avalonia", "Gitter" diff --git a/build/common.props b/build/common.props index a9bd13f812..6fc04d2938 100644 --- a/build/common.props +++ b/build/common.props @@ -23,24 +23,29 @@ true annotations + + true - + $(MSBuildThisFileDirectory)strongNameKey.snk true + + + true - 11.0 + 12.0 - 0.13.10 + 0.15.4 @@ -56,12 +61,13 @@ - + all runtime; build; native; contentfiles; analyzers - + all + runtime; build; native; contentfiles; analyzers diff --git a/build/sdk/global.json b/build/sdk/global.json index c1e852daeb..be5c357f30 100644 --- a/build/sdk/global.json +++ b/build/sdk/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "7.0.401", + "version": "8.0.410", "rollForward": "disable" } } diff --git a/build/versions.txt b/build/versions.txt index e202cbc55a..4b22ce0ad0 100644 --- a/build/versions.txt +++ b/build/versions.txt @@ -54,4 +54,12 @@ 0.13.7 0.13.8 0.13.9 -0.13.10 \ No newline at end of file +0.13.10 +0.13.11 +0.13.12 +0.14.0 +0.15.0 +0.15.1 +0.15.2 +0.15.3 +0.15.4 \ No newline at end of file diff --git a/docs/.gitignore b/docs/.gitignore index aad3551319..ebe91edcfe 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -11,4 +11,6 @@ _exported_templates docfx-bin api/* !api/index.md -/index.md \ No newline at end of file +/index.md +_changelog +changelog diff --git a/docs/_changelog/.gitignore b/docs/_changelog/.gitignore deleted file mode 100644 index 3bc49a8a60..0000000000 --- a/docs/_changelog/.gitignore +++ /dev/null @@ -1 +0,0 @@ -details diff --git a/docs/_changelog/footer/v0.10.0.md b/docs/_changelog/footer/v0.10.0.md deleted file mode 100644 index 823ad6954c..0000000000 --- a/docs/_changelog/footer/v0.10.0.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: November 10, 2016_ - -_Milestone: [v0.10.0](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.0)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.0 - -_Online Documentation:_ https://dotnet.github.io/BenchmarkDotNet/ \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.1.md b/docs/_changelog/footer/v0.10.1.md deleted file mode 100644 index 6a2f16d19d..0000000000 --- a/docs/_changelog/footer/v0.10.1.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: December 04, 2016_ - -_Milestone: [v0.10.1](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.1)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.1 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.10.md b/docs/_changelog/footer/v0.10.10.md deleted file mode 100644 index 9dde0a1cc3..0000000000 --- a/docs/_changelog/footer/v0.10.10.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: November 03, 2017_ - -_Milestone: [v0.10.10](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.10)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.9...v0.10.10)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.10 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.11.md b/docs/_changelog/footer/v0.10.11.md deleted file mode 100644 index 0de3715f93..0000000000 --- a/docs/_changelog/footer/v0.10.11.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: December 01, 2017_ - -_Milestone: [v0.10.11](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.11)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.10...v0.10.11)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.11 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.12.md b/docs/_changelog/footer/v0.10.12.md deleted file mode 100644 index 3106385452..0000000000 --- a/docs/_changelog/footer/v0.10.12.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: January 15, 2018_ - -_Milestone: [v0.10.12](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.12)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.11...v0.10.12)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.12 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.13.md b/docs/_changelog/footer/v0.10.13.md deleted file mode 100644 index 6eb70e1fc2..0000000000 --- a/docs/_changelog/footer/v0.10.13.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: March 02, 2018_ - -_Milestone: [v0.10.13](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.13)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.12...v0.10.13)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.13 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.13 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.13 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.13 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.14.md b/docs/_changelog/footer/v0.10.14.md deleted file mode 100644 index 2fd3ea5542..0000000000 --- a/docs/_changelog/footer/v0.10.14.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: April 09, 2018_ - -_Milestone: [v0.10.14](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.14)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.13...v0.10.14)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.14 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.14 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.14 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.14 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.2.md b/docs/_changelog/footer/v0.10.2.md deleted file mode 100644 index 46e50a038f..0000000000 --- a/docs/_changelog/footer/v0.10.2.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: January 21, 2017_ - -_Milestone: [v0.10.2](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.2)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.2 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.3.md b/docs/_changelog/footer/v0.10.3.md deleted file mode 100644 index 623439e439..0000000000 --- a/docs/_changelog/footer/v0.10.3.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: March 01, 2017_ - -_Milestone: [v0.10.3](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.3)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.3 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.4.md b/docs/_changelog/footer/v0.10.4.md deleted file mode 100644 index 9f497b878b..0000000000 --- a/docs/_changelog/footer/v0.10.4.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: April 21, 2017_ - -_Milestone: [v0.10.4](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.4)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.4 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.5.md b/docs/_changelog/footer/v0.10.5.md deleted file mode 100644 index b4d8d5b914..0000000000 --- a/docs/_changelog/footer/v0.10.5.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: April 26, 2017_ - -_Milestone: [v0.10.5](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.5)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.5 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.6.md b/docs/_changelog/footer/v0.10.6.md deleted file mode 100644 index b29dfc3939..0000000000 --- a/docs/_changelog/footer/v0.10.6.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: May 12, 2017_ - -_Milestone: [v0.10.6](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.6)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.6 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.7.md b/docs/_changelog/footer/v0.10.7.md deleted file mode 100644 index 5ee21fee21..0000000000 --- a/docs/_changelog/footer/v0.10.7.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: June 05, 2017_ - -_Milestone: [v0.10.7](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.7)_ - -_Overview post: https://aakinshin.net/posts/bdn-v0_10_7/_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.7 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.8.md b/docs/_changelog/footer/v0.10.8.md deleted file mode 100644 index 791022d425..0000000000 --- a/docs/_changelog/footer/v0.10.8.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: June 09, 2017_ - -_Milestone: [v0.10.8](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.8)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.8 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.9.md b/docs/_changelog/footer/v0.10.9.md deleted file mode 100644 index 8fc3fef03e..0000000000 --- a/docs/_changelog/footer/v0.10.9.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: July 28, 2017_ - -_Milestone: [v0.10.9](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.9)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.9 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.0.md b/docs/_changelog/footer/v0.11.0.md deleted file mode 100644 index f9637a79e6..0000000000 --- a/docs/_changelog/footer/v0.11.0.md +++ /dev/null @@ -1,8 +0,0 @@ -_Date: July 23, 2018_ - -_Milestone: [v0.11.0](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.11.0)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.14...v0.11.0)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.0 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.1.md b/docs/_changelog/footer/v0.11.1.md deleted file mode 100644 index f6c73f9510..0000000000 --- a/docs/_changelog/footer/v0.11.1.md +++ /dev/null @@ -1,8 +0,0 @@ -_Date: August 22, 2018_ - -_Milestone: [v0.11.1](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.11.1)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.0...v0.11.1)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.1 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.2.md b/docs/_changelog/footer/v0.11.2.md deleted file mode 100644 index 338170b8ae..0000000000 --- a/docs/_changelog/footer/v0.11.2.md +++ /dev/null @@ -1,8 +0,0 @@ -_Date: November 1, 2018_ - -_Milestone: [v0.11.2](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.11.2)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.1...v0.11.2)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.2 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.3.md b/docs/_changelog/footer/v0.11.3.md deleted file mode 100644 index befc032994..0000000000 --- a/docs/_changelog/footer/v0.11.3.md +++ /dev/null @@ -1,8 +0,0 @@ -_Date: November 20, 2018_ - -_Milestone: [v0.11.3](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.11.3)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.2...v0.11.3)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.3 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.4.md b/docs/_changelog/footer/v0.11.4.md deleted file mode 100644 index 5cbac89988..0000000000 --- a/docs/_changelog/footer/v0.11.4.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: February 15, 2019_ - -_Milestone: [v0.11.4](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.11.4)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.3...v0.11.4)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Tool/0.11.4 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.5.md b/docs/_changelog/footer/v0.11.5.md deleted file mode 100644 index be3282de78..0000000000 --- a/docs/_changelog/footer/v0.11.5.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: April 2, 2019_ - -_Milestone: [v0.11.5](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.11.5)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.4...v0.11.5)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Tool/0.11.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.11.5 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.12.0.md b/docs/_changelog/footer/v0.12.0.md deleted file mode 100644 index 565429ca6a..0000000000 --- a/docs/_changelog/footer/v0.12.0.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: October 24, 2019_ - -_Milestone: [v0.12.0](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.12.0)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.5...v0.12.0)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.12.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.12.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Tool/0.12.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.12.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.12.0 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.12.1.md b/docs/_changelog/footer/v0.12.1.md deleted file mode 100644 index af5fca9dc4..0000000000 --- a/docs/_changelog/footer/v0.12.1.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: April 6, 2020_ - -_Milestone: [v0.12.1](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.12.1)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.12.0...v0.12.1)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.12.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.12.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Tool/0.12.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.12.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.12.1 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.0.md b/docs/_changelog/footer/v0.13.0.md deleted file mode 100644 index 7a8ed327bc..0000000000 --- a/docs/_changelog/footer/v0.13.0.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: May 19, 2021_ - -_Milestone: [v0.13.0](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.0)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.12.1...v0.13.0)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.0 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.1.md b/docs/_changelog/footer/v0.13.1.md deleted file mode 100644 index eb71d8729f..0000000000 --- a/docs/_changelog/footer/v0.13.1.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: August 11, 2021_ - -_Milestone: [v0.13.1](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.1)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.0...v0.13.1)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.1 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.10.md b/docs/_changelog/footer/v0.13.10.md deleted file mode 100644 index 68ef4fd0db..0000000000 --- a/docs/_changelog/footer/v0.13.10.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: November 01, 2023_ - -_Milestone: [v0.13.10](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.10)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.9...v0.13.10)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.10 diff --git a/docs/_changelog/footer/v0.13.2.md b/docs/_changelog/footer/v0.13.2.md deleted file mode 100644 index c9022e590e..0000000000 --- a/docs/_changelog/footer/v0.13.2.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: August 26, 2022_ - -_Milestone: [v0.13.2](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.2)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.1...v0.13.2)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.2 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.3.md b/docs/_changelog/footer/v0.13.3.md deleted file mode 100644 index f0fa6dff23..0000000000 --- a/docs/_changelog/footer/v0.13.3.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: December 26, 2022_ - -_Milestone: [v0.13.3](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.3)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.2...v0.13.3)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.3 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.4.md b/docs/_changelog/footer/v0.13.4.md deleted file mode 100644 index b6ac2ad6af..0000000000 --- a/docs/_changelog/footer/v0.13.4.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: January 13, 2023_ - -_Milestone: [v0.13.4](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.4)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.3...v0.13.4)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.4 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.5.md b/docs/_changelog/footer/v0.13.5.md deleted file mode 100644 index 577eeaea84..0000000000 --- a/docs/_changelog/footer/v0.13.5.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: February 17, 2023_ - -_Milestone: [v0.13.5](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.5)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.4...v0.13.5)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.5 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.6.md b/docs/_changelog/footer/v0.13.6.md deleted file mode 100644 index 7235cd2302..0000000000 --- a/docs/_changelog/footer/v0.13.6.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: July 11, 2023_ - -_Milestone: [v0.13.6](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.6)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.5...v0.13.6)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.6 diff --git a/docs/_changelog/footer/v0.13.7.md b/docs/_changelog/footer/v0.13.7.md deleted file mode 100644 index 1b256837d4..0000000000 --- a/docs/_changelog/footer/v0.13.7.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: August 04, 2023_ - -_Milestone: [v0.13.7](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.7)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.6...v0.13.7)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.7 diff --git a/docs/_changelog/footer/v0.13.8.md b/docs/_changelog/footer/v0.13.8.md deleted file mode 100644 index a8551cc47f..0000000000 --- a/docs/_changelog/footer/v0.13.8.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: September 08, 2023_ - -_Milestone: [v0.13.8](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.8)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.7...v0.13.8)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.8 diff --git a/docs/_changelog/footer/v0.13.9.md b/docs/_changelog/footer/v0.13.9.md deleted file mode 100644 index 962bc252cc..0000000000 --- a/docs/_changelog/footer/v0.13.9.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: October 05, 2023_ - -_Milestone: [v0.13.9](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.9)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.8...v0.13.9)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.9 diff --git a/docs/_changelog/footer/v0.8.2.md b/docs/_changelog/footer/v0.8.2.md deleted file mode 100644 index cbd405d3ae..0000000000 --- a/docs/_changelog/footer/v0.8.2.md +++ /dev/null @@ -1,4 +0,0 @@ -_Date: January 19, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.8.2 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.0.md b/docs/_changelog/footer/v0.9.0.md deleted file mode 100644 index 55346f6951..0000000000 --- a/docs/_changelog/footer/v0.9.0.md +++ /dev/null @@ -1,4 +0,0 @@ -_Date: February 9, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.0 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.1.md b/docs/_changelog/footer/v0.9.1.md deleted file mode 100644 index b38f8a6436..0000000000 --- a/docs/_changelog/footer/v0.9.1.md +++ /dev/null @@ -1,4 +0,0 @@ -_Date: February 10, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.1 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.2.md b/docs/_changelog/footer/v0.9.2.md deleted file mode 100644 index 73c081a4cd..0000000000 --- a/docs/_changelog/footer/v0.9.2.md +++ /dev/null @@ -1,6 +0,0 @@ -_Milestone: [v0.9.2](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.2)_ - -_Date: March 5, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.2 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.3.md b/docs/_changelog/footer/v0.9.3.md deleted file mode 100644 index da9b45b165..0000000000 --- a/docs/_changelog/footer/v0.9.3.md +++ /dev/null @@ -1,7 +0,0 @@ -_Milestone: [v0.9.3](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.3)_ - -_Date: March 13, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.3 -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.3-beta \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.4.md b/docs/_changelog/footer/v0.9.4.md deleted file mode 100644 index 0927d0a62b..0000000000 --- a/docs/_changelog/footer/v0.9.4.md +++ /dev/null @@ -1,7 +0,0 @@ -_Milestone: [v0.9.4](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.4)_ - -_Date: March 24, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.4 -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.4-beta \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.5.md b/docs/_changelog/footer/v0.9.5.md deleted file mode 100644 index ea7d920994..0000000000 --- a/docs/_changelog/footer/v0.9.5.md +++ /dev/null @@ -1,7 +0,0 @@ -_Milestone: [v0.9.5](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.5)_ - -_Date: May 02, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.5 -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.5-beta \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.6.md b/docs/_changelog/footer/v0.9.6.md deleted file mode 100644 index cc71d2d927..0000000000 --- a/docs/_changelog/footer/v0.9.6.md +++ /dev/null @@ -1,8 +0,0 @@ -_Milestone: [v0.9.6](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.6)_ - -_Date: May 11, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.6 -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.6-beta -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.9.6 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.7.md b/docs/_changelog/footer/v0.9.7.md deleted file mode 100644 index 21f1d109e5..0000000000 --- a/docs/_changelog/footer/v0.9.7.md +++ /dev/null @@ -1,8 +0,0 @@ -_Milestone: [v0.9.7](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.7)_ - -_Date: May 29, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.7 -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.7-beta -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.9.7 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.8.md b/docs/_changelog/footer/v0.9.8.md deleted file mode 100644 index fa5bc33931..0000000000 --- a/docs/_changelog/footer/v0.9.8.md +++ /dev/null @@ -1,7 +0,0 @@ -_Milestone: [v0.9.8](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.8)_ - -_Date: July 07, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.9.8 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.9.md b/docs/_changelog/footer/v0.9.9.md deleted file mode 100644 index 2e6b340c90..0000000000 --- a/docs/_changelog/footer/v0.9.9.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: August 18, 2016_ - -_Milestone: [v0.9.9](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.9)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.9.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.9.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.9.9 - -_Online Documentation:_ https://perfdotnet.github.io/BenchmarkDotNet/ \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.0.md b/docs/_changelog/header/v0.10.0.md deleted file mode 100644 index 48c4103b0a..0000000000 --- a/docs/_changelog/header/v0.10.0.md +++ /dev/null @@ -1,12 +0,0 @@ -* Now BenchmarkDotNet is a part of .NET Foundation -* Job and Column API refactoring (see new documentation) -* Measurement engine improvements -* Horology enhancement (see `TimeInterval` and `Frequency`) -* Introduced `RankColumn` which is based on `WelchTTest` (see [157aabc3](https://github.com/PerfDotNet/BenchmarkDotNet/commit/cf839a0d7ecfdf93da709b63fe324fd2157aabc3)) -* JsonExporters refactoring (see the Exporters/Json section in the documentation) - * Renamed JsonExporters classed and attributes - * JsonExporters with custom settings - * JsonExporters now includes information about the target type namespace (see [#246](https://github.com/PerfDotNet/BenchmarkDotNet/issues/246)). -* Add `JetBrains.Annotations` (see [#253](https://github.com/PerfDotNet/BenchmarkDotNet/pull/253)) -* RFC 4180 support in CSV exporters (see [#241](https://github.com/PerfDotNet/BenchmarkDotNet/issues/241)) -* Many bugfixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.1.md b/docs/_changelog/header/v0.10.1.md deleted file mode 100644 index 70e3be82c8..0000000000 --- a/docs/_changelog/header/v0.10.1.md +++ /dev/null @@ -1,8 +0,0 @@ -* MemoryDiagnoser got improved. The changes: - * Memory Diagnoser is now part of BenchmarkDotNet.Core.dll, and it's **enabled by default** - * MemoryDiagnoser is **100% accurate** about allocated memory when using default settings or Job.ShortRun or any longer job. (see [#284](https://github.com/dotnet/BenchmarkDotNet/pull/284)) - * Memory Diagnoser no longer includes allocations from Cleanup/Setup methods (see [#186](https://github.com/dotnet/BenchmarkDotNet/issues/186)) - * the results are now scaled so they are stable across the runs. (see [#133](https://github.com/dotnet/BenchmarkDotNet/issues/133)) -* .NET Core 1.1+ support, we no longer support 1.0, we target netcoreapp1.1 now. Reason: we wanted to use `GC.GetAllocatedBytesForCurrentThread` in MemoryDiagnoser which is available only in 1.1+ -* Improved information about environment in summary -* Minor bugfixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.10.md b/docs/_changelog/header/v0.10.10.md deleted file mode 100644 index ef34841958..0000000000 --- a/docs/_changelog/header/v0.10.10.md +++ /dev/null @@ -1,10 +0,0 @@ -Highlights: - -* Disassembly Diagnoser (read more here: [Disassembling .NET Code with BenchmarkDotNet](https://adamsitnik.com/Disassembly-Diagnoser/)) -* ParamsSources -* .NET Core x86 support -* Environment variables and Mono args support -* Better environment description -* More: additional sections in the documentation, bug fixes, build script improvements, internal refactoring. - -Overview post: [BenchmarkDotNet v0.10.10](https://aakinshin.net/posts/bdn-v0_10_10/) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.11.md b/docs/_changelog/header/v0.10.11.md deleted file mode 100644 index 3e53cf54cb..0000000000 --- a/docs/_changelog/header/v0.10.11.md +++ /dev/null @@ -1,8 +0,0 @@ -## Highlights - -* ByRef and Stack-only support ([#492](https://github.com/dotnet/BenchmarkDotNet/issues/492), [sample](https://github.com/dotnet/BenchmarkDotNet/blob/edf20758871ec621fdbfd93d862769da46c4bf15/samples/BenchmarkDotNet.Samples/IL/IL_RefReturns.cs)) -* .NET Core 2.1 support ([#587](https://github.com/dotnet/BenchmarkDotNet/issues/587)) -* Improved LINQPad support -* Smart logic for precision in ScaledColumn ([#509](https://github.com/dotnet/BenchmarkDotNet/issues/509), [#590](https://github.com/dotnet/BenchmarkDotNet/issues/590)) -* Better macOS version detection ([15d72388](https://github.com/dotnet/BenchmarkDotNet/commit/15d72388436c1060e87662b5f4519b9e7e071627)) -* Minor fixes and improvements \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.12.md b/docs/_changelog/header/v0.10.12.md deleted file mode 100644 index 43cff33827..0000000000 --- a/docs/_changelog/header/v0.10.12.md +++ /dev/null @@ -1,22 +0,0 @@ -Overview post: [BenchmarkDotNet v0.10.12](https://aakinshin.net/posts/bdn-v0_10_12/) - -### Highlights - -* **Improved DisassemblyDiagnoser:** - BenchmarkDotNet contains an embedded disassembler so that it can print assembly code for all benchmarks; - it's not easy, but the disassembler evolves in every release. -* **Improved MemoryDiagnoser:** - it has a better precision level, and it takes less time to evaluate memory allocations in a benchmark. -* **New TailCallDiagnoser:** - now you get notifications when JIT applies the tail call optimizations to your methods. -* **Better environment info:** - when your share performance results, it's very important to share information about your environment. - The library generates the environment summary for you by default. - Now it contains information about the amount of physical CPU, physical cores, and logic cores. - If you run a benchmark on a virtual machine, you will get the name of the hypervisor - (e.g., Hyper-V, VMware, or VirtualBox). -* **Better summary table:** - one of the greatest features of BenchmarkDotNet is the summary table. - It shows all important information about results in a compact and understandable form. - Now it has better customization options: you can display relative performance of different environments - (e.g., compare .NET Framework and .NET Core) and group benchmarks by categories. \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.13.md b/docs/_changelog/header/v0.10.13.md deleted file mode 100644 index 73f5aa1e53..0000000000 --- a/docs/_changelog/header/v0.10.13.md +++ /dev/null @@ -1 +0,0 @@ -Overview post: [BenchmarkDotNet v0.10.13](https://aakinshin.net/posts/bdn-v0_10_13/) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.14.md b/docs/_changelog/header/v0.10.14.md deleted file mode 100644 index aefb183459..0000000000 --- a/docs/_changelog/header/v0.10.14.md +++ /dev/null @@ -1,4 +0,0 @@ -* Per-method parameterization ([Read more](https://benchmarkdotnet.org/articles/features/parameterization.html)) -* Console histograms and multimodal disribution detection -* Many improvements for Mono disassembly support on Windows ([Read more](https://aakinshin.net/posts/dotnet-crossruntime-disasm/)) -* Many bugfixes diff --git a/docs/_changelog/header/v0.10.2.md b/docs/_changelog/header/v0.10.2.md deleted file mode 100644 index a93c8c36ae..0000000000 --- a/docs/_changelog/header/v0.10.2.md +++ /dev/null @@ -1,9 +0,0 @@ -* Closed [#307](https://github.com/dotnet/BenchmarkDotNet/issues/307): culture invariant statistics output -* Closed [#321](https://github.com/dotnet/BenchmarkDotNet/issues/321): persist optimized, auto-generated dll compiled from url/plain code -* Closed [#322](https://github.com/dotnet/BenchmarkDotNet/issues/332): always restore the console foreground color -* Closed [#337](https://github.com/dotnet/BenchmarkDotNet/issues/337): Better detection of Rscript.exe in RPlotExporter -* Closed [#345](https://github.com/dotnet/BenchmarkDotNet/issues/345): fix bug in WelchTTestPValueColumn for DryJob -* VS 2017 [compatibility fix](https://github.com/dotnet/BenchmarkDotNet/commit/f4bdae5b7e203e0f0a7d283db5faa78107674f31) -* [fix](https://github.com/dotnet/BenchmarkDotNet/commit/24dea483b8312efba669d82a6fac3603e60050f5) bold markup for Atlassian exporter -* Improved precision of nanobenchmarks -* Minor infrastructure changes and misc fixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.3.md b/docs/_changelog/header/v0.10.3.md deleted file mode 100644 index 976a78eeaf..0000000000 --- a/docs/_changelog/header/v0.10.3.md +++ /dev/null @@ -1,8 +0,0 @@ -* **New .csprojs** support for .NET Core. Also for F# ([#366](https://github.com/dotnet/BenchmarkDotNet/issues/366))! -* New plots and RPlotExporter (density plots for each job; cumulative mean plots) -* Fixed exporter order (now RPlotExporer uses the actual measurements instead of previous version) -* Xplat improvments in RuntimeInformation -* Introduced `RunStrategy.Monitoring` -* Possibility to set custom path for Mono ([#306](https://github.com/dotnet/BenchmarkDotNet/issues/306)) -* Possibility to set any .NET Core version >= 1.1 ([#336](https://github.com/dotnet/BenchmarkDotNet/issues/336)) -* **MemoryDiagnoser is now disabled by default (Breaking changes!!)** ([#369](https://github.com/dotnet/BenchmarkDotNet/issues/369)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.4.md b/docs/_changelog/header/v0.10.4.md deleted file mode 100644 index f423a6567a..0000000000 --- a/docs/_changelog/header/v0.10.4.md +++ /dev/null @@ -1,23 +0,0 @@ -* New logo -* Update to Roslyn 2.0, drop .NET 4.5 support ([#303](https://github.com/dotnet/BenchmarkDotNet/issues/303)) -* Initial support of HardwareCounters (Windows only) -* Initial experimental support of in-process benchmarks -* Optional configs for `BenchmarkSwitcher` ([#391](https://github.com/dotnet/BenchmarkDotNet/issues/391), [#392](https://github.com/dotnet/BenchmarkDotNet/pull/392)) -* Host API interface ([#356](https://github.com/dotnet/BenchmarkDotNet/pull/356)) -* Improved measurements for async benchmarks ([#415](https://github.com/dotnet/BenchmarkDotNet/issues/415)) -* Improved precision level (MinIterationTimes is 500ms instead of 200ms; introduced `AccuracyMode.MaxAbsoluteError` and `AccuracyMode.MaxRelativeError` instead of `AccuracyMode.MaxStdErrRelative`; logic which select amount of iterations uses confidence intervals instead of standard errors; the Error column (half of CI99.9%) is shown by default instead of StdErr) -* Introduced `ISummaryStyle`, raw data in CSV reports ([#118](https://github.com/dotnet/BenchmarkDotNet/issues/118), [#146](https://github.com/dotnet/BenchmarkDotNet/issues/146), [#396](https://github.com/dotnet/BenchmarkDotNet/pull/396)) -* Handle cases when report files are existed and locked ([#414](https://github.com/dotnet/BenchmarkDotNet/issues/414), [#416](https://github.com/dotnet/BenchmarkDotNet/pull/416)) -* MarkdownExporter right-justifies numeric columns ([#421](https://github.com/dotnet/BenchmarkDotNet/pull/421)) -* Better colors for console output ([#376](https://github.com/dotnet/BenchmarkDotNet/issues/376)) -* Column legends -* Add information about CPU microarchitecture for well-known processors to summary -* Fix AssemblyInformationalVersionAttribute ([#382](https://github.com/dotnet/BenchmarkDotNet/issues/382)) -* Fix incorrect method filtering in BenchmarkSwitcher ([#365](https://github.com/dotnet/BenchmarkDotNet/issues/365)) -* Fix OS Version in Summary for Windows 10 ([#351](https://github.com/dotnet/BenchmarkDotNet/issues/351)) -* Fix OS Version on Mono -* Fix --class and --method filtering ([#249](https://github.com/dotnet/BenchmarkDotNet/issues/249)) -* Fix --exporters option ([#189](https://github.com/dotnet/BenchmarkDotNet/issues/189)) -* Fix escaping logic in CsvExporter ([#294](https://github.com/dotnet/BenchmarkDotNet/issues/294), [#409](https://github.com/dotnet/BenchmarkDotNet/pull/409)) -* [Fix](https://github.com/dotnet/BenchmarkDotNet/commit/0a251b81b826876179740cc8b79c994a73a5cd51) MacOS detection -* Minor bugfixes and API improvements \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.5.md b/docs/_changelog/header/v0.10.5.md deleted file mode 100644 index 3ed3541c70..0000000000 --- a/docs/_changelog/header/v0.10.5.md +++ /dev/null @@ -1,5 +0,0 @@ -* Fixed SizeUnit presentation in the summary table ([#434](https://github.com/dotnet/BenchmarkDotNet/issues/434)) -* In MemoryDiagnoser, now 1kB = 1024B (instead of 1000 in v0.10.4) ([#434](https://github.com/dotnet/BenchmarkDotNet/issues/434)) -* Fix false allocations detection ([#436](https://github.com/dotnet/BenchmarkDotNet/pull/436) [9b44de70](https://github.com/dotnet/BenchmarkDotNet/commit/9b44de704b96e2333d762b14daa152d859b1917d)) -* Hide ScaledSD column for small values ([da857ad7](https://github.com/dotnet/BenchmarkDotNet/commit/da857ad7eda77db813692d3c3678f8ad04f5af78)) -* Autoselecting amount of digits after the decimal point ([#404](https://github.com/dotnet/BenchmarkDotNet/issues/404)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.6.md b/docs/_changelog/header/v0.10.6.md deleted file mode 100644 index 33948b2f80..0000000000 --- a/docs/_changelog/header/v0.10.6.md +++ /dev/null @@ -1,6 +0,0 @@ -* Removed buggy allocation from Engine which was spoiling the results of MemoryDiagnoser for micro benchmarks. This part of the code is now guarded with very strict integration tests, it should never happen again. We now also exclude the side effects of the Allocation Quantum. **This bug was serious, you must update to `0.10.6`** ([#439](https://github.com/dotnet/BenchmarkDotNet/issues/439)) -* Support of the `PackageTargetFallback` setting which allows to reference components that target old framework monikers (like `dotnet5.4` or `portable-net45+win8`) ([#438](https://github.com/dotnet/BenchmarkDotNet/issues/438)) -* Added `InstructionRetiredPerCycleColumn` which shows up automatically when `HardwareCounter.InstructionRetired` and `HardwareCounter.TotalCycles` are used. -* Support benchmark classes without namespace ([#446](https://github.com/dotnet/BenchmarkDotNet/issues/446)) -* Fix problem with RPlotExporter and quoted directories in %PATH% ([#446](https://github.com/dotnet/BenchmarkDotNet/issues/446)) -* Show Windows brand version in summary \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.7.md b/docs/_changelog/header/v0.10.7.md deleted file mode 100644 index e58ad7f60e..0000000000 --- a/docs/_changelog/header/v0.10.7.md +++ /dev/null @@ -1,6 +0,0 @@ -* LINQPad support (5.22.05+) ([#66](https://github.com/dotnet/BenchmarkDotNet/issues/66), [#445](https://github.com/dotnet/BenchmarkDotNet/issues/445)) -* Benchmark filters and categories ([#248](https://github.com/dotnet/BenchmarkDotNet/issues/248)) -* Updated setup/cleanup attributes: `[GlobalSetup]`, `[GlobalCleanup]`, `[IterationSetup]`, `[IterationCleanup]` ([#270](https://github.com/dotnet/BenchmarkDotNet/issues/270), [#274](https://github.com/dotnet/BenchmarkDotNet/issues/274), [#325](https://github.com/dotnet/BenchmarkDotNet/issues/325), [#456](https://github.com/dotnet/BenchmarkDotNet/issues/456)) -* Better Value Types support ([afa803d0](https://github.com/dotnet/BenchmarkDotNet/commit/afa803d0e38c0e11864b2e4394d4a85d3801d944)) -* Building Sources on Linux: it's possible to build the solution (with unloaded F#/VB projects), run samples (for both net46/netcoreapp1.1), run unit tests (for netcoreapp1.1 only) -* Fix minor bugs in `JsonExporter` ([#451](https://github.com/dotnet/BenchmarkDotNet/pull/451)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.8.md b/docs/_changelog/header/v0.10.8.md deleted file mode 100644 index c93b24ee6e..0000000000 --- a/docs/_changelog/header/v0.10.8.md +++ /dev/null @@ -1,4 +0,0 @@ -* Legend for time units ([#349](https://github.com/dotnet/BenchmarkDotNet/issues/349), [#459](https://github.com/dotnet/BenchmarkDotNet/issues/459), [f14e508e](https://github.com/dotnet/BenchmarkDotNet/commit/f14e508e44b510a26cc3ec5aed30ee7843a92baf)) -* XML exporter ([#157](https://github.com/dotnet/BenchmarkDotNet/issues/157), [#452](https://github.com/dotnet/BenchmarkDotNet/pull/452), [a0148db8](https://github.com/dotnet/BenchmarkDotNet/commit/a0148db80c518a9d255f496534a8d1666be52c69)) -* .NET Framework 4.7 support ([#461](https://github.com/dotnet/BenchmarkDotNet/issues/461), [3f2b5c3c](https://github.com/dotnet/BenchmarkDotNet/commit/3f2b5c3c134c62f34f0ecf1a9c90d91ad37f2c6a), [5513873a](https://github.com/dotnet/BenchmarkDotNet/commit/5513873ac40d07583d6136e431e3b7c8cdf6c851)) -* Public API for AllocationQuantum ([#450](https://github.com/dotnet/BenchmarkDotNet/issues/450), [#462](https://github.com/dotnet/BenchmarkDotNet/pull/462), [a0148db8](https://github.com/dotnet/BenchmarkDotNet/commit/a0148db80c518a9d255f496534a8d1666be52c69)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.9.md b/docs/_changelog/header/v0.10.9.md deleted file mode 100644 index 248e6b007e..0000000000 --- a/docs/_changelog/header/v0.10.9.md +++ /dev/null @@ -1,14 +0,0 @@ -* Migrate from custom build scripts to Cake (C# Make) ([#426](https://github.com/dotnet/BenchmarkDotNet/issues/426), [#475](https://github.com/dotnet/BenchmarkDotNet/pull/475), thanks [@Ky7m](https://github.com/Ky7m)) -* Target Setup methods for specific Benchmarks ([#469](https://github.com/dotnet/BenchmarkDotNet/issues/469), [#501](https://github.com/dotnet/BenchmarkDotNet/pull/501), thanks [@ipjohnson](https://github.com/ipjohnson)) -* Many improvements in XmlExporter ([#476](https://github.com/dotnet/BenchmarkDotNet/pull/476), [#488](https://github.com/dotnet/BenchmarkDotNet/pull/488), thanks [@Teknikaali](https://github.com/Teknikaali)) -* Add MemoryDiagnoser results to JsonExporter output ([#453](https://github.com/dotnet/BenchmarkDotNet/issues/453), [#478](https://github.com/dotnet/BenchmarkDotNet/pull/478), thanks [@Teknikaali](https://github.com/Teknikaali)) -* Detect correct version of .NET Core (+ improved presentation for information about runtime) ([#448](https://github.com/dotnet/BenchmarkDotNet/issues/448), [ed586585...ed586585](https://github.com/dotnet/BenchmarkDotNet/compare/dc6dc411b4d8703d0a1abafe64fb1e0b0a83af1f...cea199f74923c99f88f4bb4d53e37f86b10269b7)) -* Fix UnauthorizedAccessException ([#380](https://github.com/dotnet/BenchmarkDotNet/issues/380), [#390](https://github.com/dotnet/BenchmarkDotNet/issues/390), [#490](https://github.com/dotnet/BenchmarkDotNet/issues/490), [#491](https://github.com/dotnet/BenchmarkDotNet/issues/491), [8505abb5](https://github.com/dotnet/BenchmarkDotNet/commit/8505abb5416bad90cda03f4972b067f9ac44b304)) -* Fix app.config generation ([#499](https://github.com/dotnet/BenchmarkDotNet/issues/499), [dc6dc411](https://github.com/dotnet/BenchmarkDotNet/commit/dc6dc411b4d8703d0a1abafe64fb1e0b0a83af1f)) -* Fix incorrect order of IterationCleanup and Benchmark jitting ([#481](https://github.com/dotnet/BenchmarkDotNet/issues/481), [#503](https://github.com/dotnet/BenchmarkDotNet/pull/503)) -* Fix test scripts for MacOS+zsh ([1177c8](https://github.com/dotnet/BenchmarkDotNet/commit/1177c80e2dbe931439e44bb0ce2ce25cad8b9ba2)) -* Unix-related ProcessorAffinity fixes ([#474](https://github.com/dotnet/BenchmarkDotNet/issues/474), [26d44411](https://github.com/dotnet/BenchmarkDotNet/commit/26d44411ea47f28a9cc7df84b2df0ef89b2bbcf7)) -* Minor fixes in docs ([#465](https://github.com/dotnet/BenchmarkDotNet/pull/465), [#467](https://github.com/dotnet/BenchmarkDotNet/pull/467), [#473](https://github.com/dotnet/BenchmarkDotNet/pull/473), [#480](https://github.com/dotnet/BenchmarkDotNet/pull/480), [#483](https://github.com/dotnet/BenchmarkDotNet/pull/483), thanks [@mtschneiders](https://github.com/mtschneiders), [@davkean](https://github.com/davkean), [@aarondandy](https://github.com/aarondandy), [@AmadeusW](https://github.com/AmadeusW)) -* Temporary hacks for [appveyor connectivity incident](https://appveyor.statuspage.io/incidents/m2vdvw39kdk8) ([#497](https://github.com/dotnet/BenchmarkDotNet/pull/497), [#506](https://github.com/dotnet/BenchmarkDotNet/pull/506)) -* Additional warnings for incorrect Configs ([#482](https://github.com/dotnet/BenchmarkDotNet/issues/482), [eb84825f](https://github.com/dotnet/BenchmarkDotNet/commit/eb84825ff08aa5d23d2d512d4d4bde3e95ca0815)) -* Additional warnings for F# methods with spaces ([#479](https://github.com/dotnet/BenchmarkDotNet/issues/479), [3c2c8dec](https://github.com/dotnet/BenchmarkDotNet/commit/3c2c8dec28d6c570f2901001058cd9c6000e6ca2), [7ba1c809](https://github.com/dotnet/BenchmarkDotNet/commit/7ba1c809004e0b75eaa87724155480eaf623f8a9), [3ca39afe](https://github.com/dotnet/BenchmarkDotNet/commit/3ca39afe9f0d25359f9b092181beb02d57c5ad32)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.11.0.md b/docs/_changelog/header/v0.11.0.md deleted file mode 100644 index 9760c809ba..0000000000 --- a/docs/_changelog/header/v0.11.0.md +++ /dev/null @@ -1,350 +0,0 @@ -This is one of the biggest releases of BenchmarkDotNet ever. -There are so many improvements. -We have - new documentation, - many performance improvements, - Job Mutators, - better user experience, - correct Ctrl+C handling, - better generic benchmarks support, - more scenarios for passing arguments to benchmarks, - awesome support of console arguments, - unicode support, - LLVM support in MonoDisassembler, - and many-many other improvements and bug fixes! - -A big part of the features and bug fixes were implemented to meet the enterprise requirements of Microsoft to make it possible to port CoreCLR, CoreFX, and CoreFXLab to BenchmarkDotNet. - -The release would not be possible without many contributions from amazing community members. This release is a combined effort. We build BenchmarkDotNet together to make benchmarking .NET code easy and available to everyone for free! - -# New documentation - -We have many improvements in our documentation! -The new docs include: - -* [DocFX](https://dotnet.github.io/docfx/) under the hood -* Detailed changelogs which includes all commits, merged pull requests and resolved issues -* API references -* Code samples for main features: - we generate it automatically based on the `BenchmarkDotNet.Samples` project; - it means that all samples can always be compiled - (no more samples with outdated API) -* Better UI -* Documentation versioning: now it's possible to look at the documentation for recent BenchmarkDotNet versions - -# Performance improvements - -BenchmarkDotNet needs to be capable of running few thousands of CoreFX and CoreCLR benchmarks in an acceptable amount of time. The code itself was already optimized so we needed architectural and design changes to meet this requirement. - -## Generate one executable per runtime settings - -To ensure that the side effects of one benchmark run does not affect another benchmark run BenchmarkDotNet generates, builds and runs every benchmark in a dedicated process. So far we were generating and building one executable per benchmark, now we generate and build one executable per runtime settings. So if you want to run ten thousands of benchmarks for .NET Core 2.1 we are going to generate and build single executable, not ten thousand. If you target multiple runtimes the build is going to be executed in parallel. Moreover, if one of the parallel builds fail it's going to be repeated in a sequential way. - -Previously the time to generate and build 650 benchmarks from our Samples project was **one hour**. Now it's something around **13 seconds** which means **276 X** improvement for this particular scenario. You can see the changes [here](https://github.com/dotnet/BenchmarkDotNet/issues/699). - -## Don't execute long operations more than once per iteration - -BenchmarkDotNet was designed to allow for very accurate and stable micro-benchmarking. One of the techniques that we use is manual loop unrolling. In practice, it meant that for every iteration we were executing the benchmark at least 16 times (the default `UnrollFactor` value). It was of course not desired for the very time-consuming benchmarks. - -So far this feature was always enabled by default and users would need to configure `UnrollFactor=1` to disable it. Now BenchmarkDotNet is going to discover such scenario and don't perform manual loop unrolling for the very time-consuming benchmarks. BenchmarkDotNet uses `Job.IterationTime` setting (the default is 0.5s) in the Pilot Experiment stage to determine how many times given benchmark should be executed per iteration. - -Example: - -```cs -public class Program -{ - static void Main() => BenchmarkRunner.Run(); - - [Benchmark] - public void Sleep1s() => Thread.Sleep(TimeSpan.FromSeconds(1)); -} -``` - -Time to run with the previous version: **374 seconds**. With `0.11.0` it's **27 seconds** which gives us almost **14 X** improvement. A good example of benchmarks that are going to benefit from this change are computer game benchmarks and ML.NET benchmarks. You can see the changes [here](https://github.com/dotnet/BenchmarkDotNet/pull/760) and [here](https://github.com/dotnet/BenchmarkDotNet/pull/771). - -## Exposing more configuration settings - -The default settings were configured to work well with every scenario. Before running the benchmark, BenchmarkDotNet does not know anything about it. This is why it performs many warmup iterations before running the benchmarks. - -When you author benchmarks and run them many times you can come up with custom settings that produce similar results but in a shorter manner of time. To allow you to do that we have exposed: - -* `Job.MinIterationCount` (default value is 15) -* `Job.MaxIterationCount` (default value is 100) -* `Job.MinWarmupIterationCount` (default value is 6) -* `Job.MaxWarmupIterationCount` (default value is 50) - -# User Experience - -One of the biggest success factors of BenchmarkDotNet is a great user experience. The tool just works as expected and makes your life easy. We want to make it even better! - -## .NET Standard 2.0 - -We have ported BenchmarkDotNet to .NET Standard 2.0 and thanks to that we were able to not only simplify our code and build process but also merge `BenchmarkDotNet.Core.dll` and `BenchmarkDotNet.Toolchains.Roslyn.dll` into `BenchmarkDotNet.dll`. We still support .NET 4.6 but we have dropped .NET Core 1.1 support. More information and full discussion can be found [here](https://github.com/dotnet/BenchmarkDotNet/pull/688). - -**Note:** Our `BenchmarkDotNet.Diagnostics.Windows` package which uses `EventTrace` to implement ETW-based diagnosers was also ported to .NET Standard 2.0 and you can now use all the ETW diagnosers with .NET Core on Windows. We plan to add EventPipe support and make this page fully cross-platform and Unix compatible soon. - -## Using complex types as benchmark arguments - -So far we have required the users to implement `IParam` interface to make the custom complex types work as benchmark arguments/parameters. This has changed, now the users can use any complex types as arguments and it will just work ([more](https://github.com/dotnet/BenchmarkDotNet/pull/754)). - -```cs -public class Program -{ - static void Main(string[] args) => BenchmarkRunner.Run(); - - public IEnumerable Arguments() - { - yield return new Point2D(10, 200); - } - - [Benchmark] - [ArgumentsSource(nameof(Arguments))] - public int WithArgument(Point2D point) => point.X + point.Y; -} - -public class Point2D -{ - public int X, Y; - - public Point2D(int x, int y) - { - X = x; - Y = y; - } - - public override string ToString() => $"[{X},{Y}]"; -} -``` - -**Note**: If you want to control what will be displayed in the summary you should override `ToString`. - -## If IterationSetup is provided run benchmark once per iteration - -When Stephen Toub says that something is [buggy](https://github.com/dotnet/BenchmarkDotNet/issues/730), it most probably is. BenchmarkDotNet performs multiple invocations of benchmark per every iteration. When we have exposed the `[IterationSetup]` attribute many users were expecting that the `IterationSetup` is going to be invoked before every benchmark execution. - -It was invoked before every iteration, and iteration was more than one benchmark call if the user did not configure that explicitly. We have changed that and now if you provide an `[IterationSetup]` method it is going to be executed before every iteration and iteration will invoke the benchmark just once. - -```cs -public class Test -{ - public static void Main() => BenchmarkRunner.Run(); - - [IterationSetup] - public void MySetup() => Console.WriteLine("MySetup"); - - [Benchmark] - public void MyBenchmark() => Console.WriteLine("MyBenchmark"); -} -``` - -Before: - -```log -MySetup -MyBenchmark -MyBenchmark -MyBenchmark -MyBenchmark -(...) -``` - -After: - -```log -MySetup -MyBenchmark -MySetup -MyBenchmark -MySetup -MyBenchmark -(...) -``` - -**Note:** If you want to configure how many times benchmark should be invoked per iteration you can use the new `[InvocationCountAttribute]`. - -## Job Mutators - -`Job` represents a set of settings to run the benchmarks. We run every benchmark for every job defined by the user. The problem was that so far many jobs were just added to the config instead of being merged with other jobs. - -An example: - -```cs -[ClrJob, CoreJob] -[GcServer(true)] -public class MyBenchmarkClass -``` - -Resulted in 3 jobs and 3 benchmark executions: `ClrJob`, `CoreJob` and `GcServer(true)` for current runtime. - -Now all Jobs and their corresponding attributes marked as mutators are going to be applied to other jobs, not just added to the config. So in this particular scenario, the benchmarks from `MyBenchmarkClass` are going to be executed for .NET with Server GC enabled and .NET Core with Server GC enabled. - -Mutators are great when you want to have a single, global config for all benchmarks and apply given settings only to selected types. You can find out more about mutators [here](https://github.com/dotnet/BenchmarkDotNet/pull/800). - -## Ctrl+C - -When the user: - -* presses `Ctrl+C` -* presses `Ctrl+Break` -* logs off -* closes console window - -We are now going to close any existing ETW session created by BenchmarkDotNet and **restore console colors** ([read more](https://github.com/dotnet/BenchmarkDotNet/pull/761)). - -## Handle OutOfMemoryException more gracefully - -When our benchmark hits `OutOfMemoryException` we print some nice explanation: - -```cs -public class Program -{ - static void Main(string[] args) => BenchmarkRunner.Run(); - - private List list = new List(); - - [Benchmark] - public void AntiPattern() => list.Add(new int[int.MaxValue / 2]); -} -``` - -```log -OutOfMemoryException! -BenchmarkDotNet continues to run additional iterations until desired accuracy level is achieved. It's possible only if the benchmark method doesn't have any side-effects. -If your benchmark allocates memory and keeps it alive, you are creating a memory leak. -You should redesign your benchmark and remove the side-effects. You can use `OperationsPerInvoke`, `IterationSetup` and `IterationCleanup` to do that. -``` - -## Trimming long strings - -We used to display the values "as is" which was bad for long strings. Now the values are trimmed ([more](https://github.com/dotnet/BenchmarkDotNet/issues/748)). - -```cs -public class Long -{ - [Params("text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7")] - public string Text; - - [Benchmark] - public int HashCode() => Text.GetHashCode(); -} -``` - -| Method | Text | -|--------- |--------------------- | -| HashCode | text/(...)q=0.7 [86] | - -# More features - -## Generic benchmarks - -BenchmarkDotNet supports generic benchmarks, all you need to do is to tell it which types should be used as generic arguments ([read more](https://github.com/dotnet/BenchmarkDotNet/pull/758)). - -```cs -[GenericTypeArguments(typeof(int))] -[GenericTypeArguments(typeof(char))] -public class IntroGenericTypeArguments -{ - [Benchmark] public T Create() => Activator.CreateInstance(); -} -``` - -## Arguments - -We now support more scenarios for passing arguments to benchmarks: - -* passing arguments to asynchronous benchmarks ([more](https://github.com/dotnet/BenchmarkDotNet/issues/818)) -* passing generic types -* passing arguments by reference -* passing jagged arrays ([more](https://github.com/dotnet/BenchmarkDotNet/issues/769)) -* types with implicit cast operator to stack only types can be passed as given stack-only types to Benchmarks ([more](https://github.com/dotnet/BenchmarkDotNet/issues/774)) - -Example: - -```cs -public class WithStringToReadOnlySpan -{ - [Benchmark] - [Arguments("some string")] - public void AcceptsReadOnlySpan(ReadOnlySpan notString) -} -``` - -## Console Arguments - -`BenchmarkSwitcher` supports various console arguments ([PR](https://github.com/dotnet/BenchmarkDotNet/pull/824)), to make it work you need to pass the `args` to switcher: - -```cs -class Program -{ - static void Main(string[] args) - => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); -} -``` - -**Note:** to get the most up-to-date info about supported console arguments run the benchmarks with `--help`. - -### Filter - -The `--filter` or just `-f` allows you to filter the benchmarks by their full name (`namespace.typeName.methodName`) using glob patterns. - -Examples: - -1. Run all benchmarks from System.Memory namespace: `-f System.Memory*` -2. Run all benchmarks: `-f *` -3. Run all benchmarks from ClassA and ClassB `-f *ClassA* *ClassB*` - -**Note**: If you would like to **join** all the results into a **single summary**, you need to use `--join`. - -### Categories - -You can also filter the benchmarks by categories: - -* `--anyCategories` - runs all benchmarks that belong to **any** of the provided categories -* `--allCategories`- runs all benchmarks that belong to **all** provided categories - -### Diagnosers - -* `-m`, `--memory` - enables MemoryDiagnoser and prints memory statistics -* `-d`, `--disassm`- enables DisassemblyDiagnoser and exports diassembly of benchmarked code - -### Runtimes - -The `--runtimes` or just `-r` allows you to run the benchmarks for selected Runtimes. Available options are: Clr, Mono, Core and CoreRT. - -Example: run the benchmarks for .NET and .NET Core: - -```log -dotnet run -c Release -- --runtimes clr core -``` - -### More arguments - -* `-j`, `--job` (Default: Default) Dry/Short/Medium/Long or Default -* `-e`, `--exporters` GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML -* `-i`, `--inProcess` (Default: false) Run benchmarks in Process -* `-a`, `--artifacts` Valid path to accessible directory -* `--outliers` (Default: OnlyUpper) None/OnlyUpper/OnlyLower/All -* `--affinity` Affinity mask to set for the benchmark process -* `--allStats` (Default: false) Displays all statistics (min, max & more) -* `--attribute` Run all methods with given attribute (applied to class or method) - -## Other small improvements - -* **Unicode support:** - now you can enable support of Unicode symbols like `μ` or `±` with `[EncodingAttribute.Unicode]`, - an example: BenchmarkDotNet.Samples.IntroEncoding - (see [#735](https://github.com/dotnet/BenchmarkDotNet/pull/735)) -* **Better benchmark validation** - (see [#693](https://github.com/dotnet/BenchmarkDotNet/pull/693), [#737](https://github.com/dotnet/BenchmarkDotNet/pull/737)) -* **Improve .NET Framework version detection**: now we support .NET Framework 4.7.2 - (see [#743](https://github.com/dotnet/BenchmarkDotNet/pull/743)) -* **OutlierModes:** - now it's possible to control how to process outliers, - an example @BenchmarkDotNet.Samples.IntroOutliers - (see [#766](https://github.com/dotnet/BenchmarkDotNet/pull/766)) -* **LLVM support in MonoDisassembler** - (see [a7426e](https://github.com/dotnet/BenchmarkDotNet/commit/a7426e84fde075503f489fdf096a95f694f77b85)) -* **Grand API renaming** - we try not to change public API, but sometimes it's necessary because we want to get a consistent and understandable API in v1.0.0. - (see [#787](https://github.com/dotnet/BenchmarkDotNet/issues/787)) -* **Many-many small improvements and bug fixes** \ No newline at end of file diff --git a/docs/_changelog/header/v0.11.1.md b/docs/_changelog/header/v0.11.1.md deleted file mode 100644 index 97efdd26d2..0000000000 --- a/docs/_changelog/header/v0.11.1.md +++ /dev/null @@ -1,11 +0,0 @@ -This release includes some minor improvements and bug fixes: - -* Fixed `RPlotExporter` ([#852](https://github.com/dotnet/BenchmarkDotNet/issues/852), [#855](https://github.com/dotnet/BenchmarkDotNet/issues/855)). - In v0.11.0, the plot generation was broken because of the huge renaming in [#787](https://github.com/dotnet/BenchmarkDotNet/issues/787). -* [ArgumentsSource](xref:BenchmarkDotNet.Samples.IntroArgumentsSource) now supports additional types like - `Type` ([#840](https://github.com/dotnet/BenchmarkDotNet/issues/840)), - `BigInteger` ([#850](https://github.com/dotnet/BenchmarkDotNet/issues/850)), - `DateTime` ([#853](https://github.com/dotnet/BenchmarkDotNet/issues/853)), - and special double values like `double.NaN` ([#851](https://github.com/dotnet/BenchmarkDotNet/issues/851)) -* Generated projects ignore Directory.Build.props and Directory.Build.targets files [#854](https://github.com/dotnet/BenchmarkDotNet/pull/854) -* Now it's possible to run benchmarks with CoreRun ([de152c](https://github.com/dotnet/BenchmarkDotNet/commit/de152c7acc71eddeaa304c846cc67e6a54ca7a0f), [#857](https://github.com/dotnet/BenchmarkDotNet/pull/857)) diff --git a/docs/_changelog/header/v0.11.2.md b/docs/_changelog/header/v0.11.2.md deleted file mode 100644 index 2e6c435a92..0000000000 --- a/docs/_changelog/header/v0.11.2.md +++ /dev/null @@ -1,657 +0,0 @@ -This release includes many PRs from the Hacktoberfest. -We are very grateful to all the contributors who spent their time to help us make BenchmarkDotNet even better! - -## Highlights - -In this release, we have many improvements in different areas: - -* **Diagnosers** - * EtwProfiler (allows profiling benchmarks on Windows and exporting the data to a trace file) -* **Execution:** - * Comparing NuGet packages (now it's possible to compare different versions of the same package) - * .NET Core 3.0 support - * Deferred Execution Validator -* **Command-line:** - * `--list`: List of benchmarks - * `--info`: Print environment info - * `--runtimes`: Choosing execution runtimes (`--runtimes net472 netcoreapp2.1` will executed a benchmark on .NET 4.7.2 and .NET Core 2.1) - * Options for number of invocations and iterations - * Custom default settings for console argument parser - * Case-insensitive filter - * Benchmarking with different CoreRun instances - * Hardware counters command-line support -* **Exporters:** - * Markdown output for DisassemblyDiagnoser - * Diff view for disassembler output - * Improved LINQPad support (colored monospaced logs) - * Better CPU brand strings -* **Attributes:** - * Async `[GlobalSetup]` and `[GlobalCleanup]` support - * Introduced `[ParamsAllValues]` - * Selecting Baseline across Methods and Jobs -* **Statistics:** - * Better statistical tests (Welch's t-test and Mann-Whitney U-test) - * ZeroMeasurementAnalyser - * RatioColumn -* **Other:** - * Azure Pipelines support for internal builds - * Many minor bug fixes - * Improved documentation - * New tests - ---- - -## Diagnosers - -### EtwProfiler - -`EtwProfiler` allows to profile the benchmarked .NET code on Windows and exports the data to a trace file which can be opened with [PerfView](https://github.com/Microsoft/perfview) or [Windows Performance Analyzer](https://learn.microsoft.com/windows-hardware/test/wpt/windows-performance-analyzer). - -`EtwProfiler` uses `TraceEvent` library which internally uses Event Tracing for Windows (ETW) to capture stack traces and important .NET Runtime events. -Before the process with benchmarked code is started, EtwProfiler starts User and Kernel ETW sessions. Every session writes data to it's own file and captures different data. User session listens for the .NET Runtime events (GC, JIT etc) while the Kernel session gets CPU stacks and Hardware Counter events. After this, the process with benchmarked code is started. During the benchmark execution all the data is captured and written to a trace file. Moreover, BenchmarkDotNet Engine emits it's own events to be able to differentiate jitting, warmup, pilot and actual workload when analyzing the trace file. When the benchmarking is over, both sessions are closed and the two trace files are merged into one. - -![](https://adamsitnik.com/images/etwprofiler/flamegraph.png) - -You can find more details - in the [documentation](xref:docs.etwprofiler) and - in the [blog post](https://adamsitnik.com/ETW-Profiler/) by Adam Sitnik. - -* [#878](https://github.com/dotnet/BenchmarkDotNet/pull/878) EtwProfiler Diagnoser (by [@adamsitnik](https://github.com/adamsitnik)) -* [04a715](https://github.com/dotnet/BenchmarkDotNet/commit/04a71586206a822bca56f0abdacefdc2e5fc1b01) EtwProfiler Diagnoser (#878) (by [@adamsitnik](https://github.com/adamsitnik)) - ---- - -## Execution - -### Comparing NuGet packages - -Now it's possible to compare performance of several versions of the same NuGet package. -An example: - -```cs -[Config(typeof(Config))] -public class IntroNuGet -{ - // Specify jobs with different versions of the same NuGet package to benchmark. - // The NuGet versions referenced on these jobs must be greater or equal to the - // same NuGet version referenced in this benchmark project. - // Example: This benchmark project references Newtonsoft.Json 9.0.1 - private class Config : ManualConfig - { - public Config() - { - var baseJob = Job.MediumRun.With(CsProjCoreToolchain.Current.Value); - Add(baseJob.WithNuGet("Newtonsoft.Json", "11.0.2").WithId("11.0.2")); - Add(baseJob.WithNuGet("Newtonsoft.Json", "11.0.1").WithId("11.0.1")); - Add(baseJob.WithNuGet("Newtonsoft.Json", "10.0.3").WithId("10.0.3")); - Add(baseJob.WithNuGet("Newtonsoft.Json", "10.0.2").WithId("10.0.2")); - Add(baseJob.WithNuGet("Newtonsoft.Json", "10.0.1").WithId("10.0.1")); - Add(baseJob.WithNuGet("Newtonsoft.Json", "9.0.1").WithId("9.0.1")); - } - } - - [Benchmark] - public void SerializeAnonymousObject() - => JsonConvert.SerializeObject( - new { hello = "world", price = 1.99, now = DateTime.UtcNow }); -} -``` - -See also: @BenchmarkDotNet.Samples.IntroNuGet - -* [#290](https://github.com/dotnet/BenchmarkDotNet/issues/290) Question: Any official way to benchmark same method between different assembly versions? -* [#931](https://github.com/dotnet/BenchmarkDotNet/issues/931) Same NuGet version used when benchmarking different packages -* [#922](https://github.com/dotnet/BenchmarkDotNet/pull/922) Enables benchmarking betweeen different Nuget packages (by [@Shazwazza](https://github.com/Shazwazza)) -* [#932](https://github.com/dotnet/BenchmarkDotNet/pull/932) Partition benchmark run info based on added nuget packages (by [@blairconrad](https://github.com/blairconrad)) -* [92a786](https://github.com/dotnet/BenchmarkDotNet/commit/92a7869aaa30aeacaf1da2dcc45bc65c8333ae73) Enables benchmarking betweeen different Nuget packages (#922) fixes #290 (by [@Shazwazza](https://github.com/Shazwazza)) -* [510685](https://github.com/dotnet/BenchmarkDotNet/commit/510685f48ce2baf57682aa82e18c6486989e9625) Partition benchmark run info based on added nuget packages (#932) (by [@blairconrad](https://github.com/blairconrad)) -* [cf84a4](https://github.com/dotnet/BenchmarkDotNet/commit/cf84a44d108d5bf3860129e0a2a78cace9c95626) NuGet casing fix (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) - -### .NET Core 3.0 support - -Now it's possible to run benchmarks on preview versions of .NET Core 3.0. - -* [#874](https://github.com/dotnet/BenchmarkDotNet/issues/874) .NET Core 3.0 support (assignee: [@adamsitnik](https://github.com/adamsitnik)) -* [2e398c](https://github.com/dotnet/BenchmarkDotNet/commit/2e398c89561b3b1c89ec64b94f656ae20236efd1) detect .NET Core 3.0 and use the appropriate target framework moniker, fixes ... (by [@adamsitnik](https://github.com/adamsitnik)) - -### Deferred Execution Validator - -In LINQ, execution of a query is usually [deferred](https://learn.microsoft.com/dotnet/standard/linq/deferred-execution-example) until the moment when you actually request the data. If your benchmark just returns `IEnumerable` or `IQueryable` it's not measuring the execution of the query, just the creation. - -This is why we decided to warn you about this issue whenever it happens: - -```log -Benchmark IntroDeferredExecution.Wrong returns a deferred execution result (IEnumerable). You need to either change the method declaration to return a materialized result or consume it on your own. You can use .Consume() extension method to do that. -``` - -Don't worry! We are also providing you with a `Consume` extension method which can execute given `IEnumerable` or `IQueryable` and consume its results. All you need to do is to create a [`Consumer`](xref:BenchmarkDotNet.Engines.Consumer) instance, preferably store it in a field (to exclude the cost of creating Consumer from the benchmark itself) and pass it to `Consume` extension method. - -**Do not call `.ToArray()` because it's an expensive operation and it might dominate given benchmark!** - -See also: @BenchmarkDotNet.Samples.IntroDeferredExecution - -* [#858](https://github.com/dotnet/BenchmarkDotNet/issues/858) Should the Engine iterate over and consume IEnumerable and IQueryable results? (assignee: [@adamsitnik](https://github.com/adamsitnik)) -* [cebe2a](https://github.com/dotnet/BenchmarkDotNet/commit/cebe2a0f84fa21acb6db9613fe3a4326d635f129) Deferred Execution Validator, fixes #858 (by [@adamsitnik](https://github.com/adamsitnik)) - ---- - -## Command-line - -In this release, we have tons of improvements for command-line experience. - -### `--list`: List of benchmarks - -The `--list` allows you to print all of the available benchmark names. Available options are: - -* `flat` - prints list of the available benchmarks: `--list flat` -```ini -BenchmarkDotNet.Samples.Algo_Md5VsSha256.Md5 -BenchmarkDotNet.Samples.Algo_Md5VsSha256.Sha256 -BenchmarkDotNet.Samples.IntroArguments.Benchmark -BenchmarkDotNet.Samples.IntroArgumentsSource.SingleArgument -BenchmarkDotNet.Samples.IntroArgumentsSource.ManyArguments -BenchmarkDotNet.Samples.IntroArrayParam.ArrayIndexOf -BenchmarkDotNet.Samples.IntroArrayParam.ManualIndexOf -BenchmarkDotNet.Samples.IntroBasic.Sleep -[...] -``` -* `tree` - prints tree of the available benchmarks: `--list tree` -```ini -BenchmarkDotNet - └─Samples - ├─Algo_Md5VsSha256 - │ ├─Md5 - │ └─Sha256 - ├─IntroArguments - │ └─Benchmark - ├─IntroArgumentsSource - │ ├─SingleArgument - │ └─ManyArguments - ├─IntroArrayParam - │ ├─ArrayIndexOf - │ └─ManualIndexOf - ├─IntroBasic - │ ├─Sleep -[...] -``` - -The `--list` option works with the `--filter` option. Examples: - -* `--list flat --filter *IntroSetupCleanup*` prints: -```ini -BenchmarkDotNet.Samples.IntroSetupCleanupGlobal.Logic -BenchmarkDotNet.Samples.IntroSetupCleanupIteration.Benchmark -BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkA -BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkB -BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkC -BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkD -``` -* `--list tree --filter *IntroSetupCleanup*` prints: -```ini -BenchmarkDotNet - └─Samples - ├─IntroSetupCleanupGlobal - │ └─Logic - ├─IntroSetupCleanupIteration - │ └─Benchmark - └─IntroSetupCleanupTarget - ├─BenchmarkA - ├─BenchmarkB - ├─BenchmarkC - └─BenchmarkD -``` - -* [#905](https://github.com/dotnet/BenchmarkDotNet/issues/905) Implement `--list` -* [#914](https://github.com/dotnet/BenchmarkDotNet/pull/914) Implement `--list` - fixes #905 (by [@wojtpl2](https://github.com/wojtpl2)) -* [#916](https://github.com/dotnet/BenchmarkDotNet/pull/916) Update console-args.md - add information about `--list` option (by [@wojtpl2](https://github.com/wojtpl2)) -* [330f66](https://github.com/dotnet/BenchmarkDotNet/commit/330f66c3a3d94d1369d5c0b629bbb0085d5db8eb) Implement `--list` - fixes #905 (#914) (by [@wojtpl2](https://github.com/wojtpl2)) -* [6c7521](https://github.com/dotnet/BenchmarkDotNet/commit/6c7521d4fd6776098667944321c8a65848382ae5) Update console-args.md - add information about `--list` option (#916) (by [@wojtpl2](https://github.com/wojtpl2)) - -### `--info`: Print environment info - -Some of our users really like the info we print about hardware and OS. -Now we have the `--info` console line argument which does not run the benchmarks, but simply prints the info. - -```ini -BenchmarkDotNet=v0.11.1.786-nightly, OS=Windows 10.0.17134.285 (1803/April2018Update/Redstone4) -Intel Xeon CPU E5-1650 v4 3.60GHz, 1 CPU, 12 logical and 6 physical cores -Frequency=3507500 Hz, Resolution=285.1033 ns, Timer=TSC -.NET Core SDK=3.0.100-alpha1-009642 - [Host] : .NET Core 3.0.0-preview1-27004-04 (CoreCLR 4.6.27003.04, CoreFX 4.6.27003.02), 64bit RyuJIT -``` - -* [#904](https://github.com/dotnet/BenchmarkDotNet/issues/904) Implement `--info` -* [#907](https://github.com/dotnet/BenchmarkDotNet/pull/907) fixes #904 Implement `--info` (by [@lahma](https://github.com/lahma)) -* [4be28d](https://github.com/dotnet/BenchmarkDotNet/commit/4be28d25fa9ab79ca194c615783148042738bdad) fixes #904 Implement `--info` (#907) (by [@lahma](https://github.com/lahma)) - -### `--runtimes`: Choosing execution runtimes - -The `--runtimes` or just `-r` allows you to run the benchmarks for selected Runtimes. Available options are: Mono, CoreRT, Core, Clr net46, net461, net462, net47, net471, net472, netcoreapp2.0, netcoreapp2.1, netcoreapp2.2, netcoreapp3.0. - -Example: run the benchmarks for .NET 4.7.2 and .NET Core 2.1: - -```log -dotnet run -c Release -- --runtimes net472 netcoreapp2.1 -``` - -* [#913](https://github.com/dotnet/BenchmarkDotNet/pull/913) .NET Core Toolchains improvements (by [@adamsitnik](https://github.com/adamsitnik)) -* [0f721c](https://github.com/dotnet/BenchmarkDotNet/commit/0f721c8e0e100fc951a54b6045eb7b58c55c2a1f) make it possible to specify runtimes using explicit tfms like net472 or netco... (by [@adamsitnik](https://github.com/adamsitnik)) -* [1c581e](https://github.com/dotnet/BenchmarkDotNet/commit/1c581e5bf5b4ba9f40d113ae09e0731a60523a60) .NET Core Toolchains improvements (#913) (by [@adamsitnik](https://github.com/adamsitnik)) - -### Options for number of invocations and iterations - -* `--launchCount` - how many times we should launch process with target benchmark. The default is 1. -* `--warmupCount` - how many warmup iterations should be performed. If you set it, the minWarmupCount and maxWarmupCount are ignored. By default calculated by the heuristic. -* `--minWarmupCount` - minimum count of warmup iterations that should be performed. The default is 6. -* `--maxWarmupCount` - maximum count of warmup iterations that should be performed. The default is 50. -* `--iterationTime` - desired time of execution of an iteration. Used by Pilot stage to estimate the number of invocations per iteration. 500ms by default. -* `--iterationCount` - how many target iterations should be performed. By default calculated by the heuristic. -* `--minIterationCount` - minimum number of iterations to run. The default is 15. -* `--maxIterationCount` - maximum number of iterations to run. The default is 100. -* `--invocationCount` - invocation count in a single iteration. By default calculated by the heuristic. -* `--unrollFactor` - how many times the benchmark method will be invoked per one iteration of a generated loop. 16 by default -* `--runOncePerIteration` - run the benchmark exactly once per iteration. False by default. - -Example: run single warmup iteration, from 9 to 12 actual workload iterations. - -```log -dotnet run -c Release -- --warmupCount 1 --minIterationCount 9 --maxIterationCount 12 -``` - -* [#902](https://github.com/dotnet/BenchmarkDotNet/pull/902) More command line args (by [@adamsitnik](https://github.com/adamsitnik)) -* [ba0d22](https://github.com/dotnet/BenchmarkDotNet/commit/ba0d22b41fd25022e3a945fe5ef1ae8aea697cf7) allow to configure the number of invocations and iterations from command line (by [@adamsitnik](https://github.com/adamsitnik)) - -### Custom default settings for console argument parser - -If you want to have a possibility to specify custom default Job settings programmatically and optionally overwrite it with console line arguments, then you should create a global config with single job marked as `.AsDefault` and pass it to `BenchmarkSwitcher` together with the console line arguments. - -Example: run single warmup iteration by default. - -```cs -static void Main(string[] args) - => BenchmarkSwitcher - .FromAssembly(typeof(Program).Assembly) - .Run(args, GetGlobalConfig()); - -static IConfig GetGlobalConfig() - => DefaultConfig.Instance - .With(Job.Default - .WithWarmupCount(1) - .AsDefault()); // the KEY to get it working -``` - -Now, the default settings are: `WarmupCount=1` but you might still overwrite it from console args like in the example below: - -```log -dotnet run -c Release -- --warmupCount 2 -``` - -### Case-insensitive filter - -The `--filter` or just `-f` allows you to filter the benchmarks by their full name (`namespace.typeName.methodName`) using glob patterns. - -Examples: - -1. Run all benchmarks from System.Memory namespace: `-f System.Memory*` -2. Run all benchmarks: `-f *` -3. Run all benchmarks from ClassA and ClassB `-f *ClassA* *ClassB*` - -Now this filter expression is case-insensitive. - -* [#864](https://github.com/dotnet/BenchmarkDotNet/issues/864) Make the filter case insensitive (assignee: [@adamsitnik](https://github.com/adamsitnik)) -* [106777](https://github.com/dotnet/BenchmarkDotNet/commit/106777f7f575a8535f16292f1de80e8ffba2853a) make the filter case insensitive invariant culture, fixes #864 (by [@adamsitnik](https://github.com/adamsitnik)) - -### Benchmarking with different CoreRun instances - -CoreRun is a simpler version of `dotnet run`, used for developing CoreCLR and CoreFX. - -Typically when working on the performance of .NET Core a developer has more than 1 copy of CoreRun. -Example: CoreRun before my changes, and after my changes. -This change allows to simply run same benchmark for few different CoreRuns to compare the perf in easy way. - -Sample usage: - -```log -dotnet run -c Release -f netcoreapp2.1 -- -f *Empty.method --job dry --coreRun -C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root\CoreRun.exe -C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root_beforeMyChanges\CoreRun.exe -``` - -Sample output: - -![image](https://user-images.githubusercontent.com/6011991/47417561-860f5800-d778-11e8-8dc6-b4e3586a9c90.png) - -* [#925](https://github.com/dotnet/BenchmarkDotNet/issues/925) Make it possible to run the benchmark with multiple CoreRun.exe (assignee: [@adamsitnik](https://github.com/adamsitnik)) -* [901616](https://github.com/dotnet/BenchmarkDotNet/commit/90161654725efd5639e0190638a3383d6a49e34c) when user provides CoreRun path and runtime in explicit way, we should use th... (by [@adamsitnik](https://github.com/adamsitnik)) -* [46bebf](https://github.com/dotnet/BenchmarkDotNet/commit/46bebf1497d4e9314c6dfd2d4e10df81332aa4fa) allow the users to run the same benchmarks using few different CoreRun.exe, f... (by [@adamsitnik](https://github.com/adamsitnik)) - -### Hardware counters command-line support - -```log ---counters CacheMisses+InstructionRetired -``` - -* [1e3df7](https://github.com/dotnet/BenchmarkDotNet/commit/1e3df74b2f927f541bed723f65c2d571fa850c53) make it possible to specify hardware counters from command line (by [@adamsitnik](https://github.com/adamsitnik)) -* [a4f91a](https://github.com/dotnet/BenchmarkDotNet/commit/a4f91a392675e4851a785095af162b977d249ba3) better handling of edge cases for parsing hardware counters from the console ... (by [@adamsitnik](https://github.com/adamsitnik)) - ---- - -## Exporters - -### Markdown output for DisassemblyDiagnoser - -Now `DisassemblyDiagnoser` generates markdown version of the assembly listing. - -* [#560](https://github.com/dotnet/BenchmarkDotNet/issues/560) Suggestion: markdown output for DisassemblyDiagnoser (assignee: [@adamsitnik](https://github.com/adamsitnik)) -* [1e6235](https://github.com/dotnet/BenchmarkDotNet/commit/1e62355f209a25c7a33f9ab7e7e03b0afe7d851f) github markdown exporter for Disassembler, fixes #560 (by [@adamsitnik](https://github.com/adamsitnik)) - -### Diff view for disassembler output - -Now we have `PrettyGithubMarkdownDiffDisassemblyExporter` which can generates - nice diffs between assembly listings. -This mode can be activated via the `--disasmDiff` command line argument or - the `printDiff: true` argument of `DisassemblyDiagnoserConfig`. -An output example (Diff between SumLocal and SumField on .NET Core 2.1.4 (CoreCLR 4.6.26814.03, CoreFX 4.6.26814.02), 64bit RyuJIT) - -```diff --; BenchmarkDotNet.Samples.IntroDisassemblyRyuJit.SumLocal() -- var local = field; // we use local variable that points to the field -- ^^^^^^^^^^^^^^^^^^ -- mov rax,qword ptr [rcx+8] -+; BenchmarkDotNet.Samples.IntroDisassemblyRyuJit.SumField() - int sum = 0; - ^^^^^^^^^^^^ -- xor edx,edx -- for (int i = 0; i < local.Length; i++) -+ xor eax,eax -+ for (int i = 0; i < field.Length; i++) - ^^^^^^^^^ -- xor ecx,ecx -- for (int i = 0; i < local.Length; i++) -+ xor edx,edx -+ for (int i = 0; i < field.Length; i++) - ^^^^^^^^^^^^^^^^ -- mov r8d,dword ptr [rax+8] -- test r8d,r8d -+ mov rcx,qword ptr [rcx+8] -+ cmp dword ptr [rcx+8],0 - jle M00_L01 -- sum += local[i]; -+ sum += field[i]; - ^^^^^^^^^^^^^^^^ - M00_L00: -- movsxd r9,ecx -- add edx,dword ptr [rax+r9*4+10h] -- for (int i = 0; i < local.Length; i++) -+ mov r8,rcx -+ cmp edx,dword ptr [r8+8] -+ jae 00007ff9`0c412c1f -+ movsxd r9,edx -+ add eax,dword ptr [r8+r9*4+10h] -+ for (int i = 0; i < field.Length; i++) - ^^^ -- inc ecx -- cmp r8d,ecx -+ inc edx -+ cmp dword ptr [rcx+8],edx - jg M00_L00 - return sum; - ^^^^^^^^^^^ - M00_L01: -- mov eax,edx --; Total bytes of code 34 -+ add rsp,28h -+; Total bytes of code 42 -``` - -* [#544](https://github.com/dotnet/BenchmarkDotNet/issues/544) Diff view for disassembler output (assignee: [@wojtpl2](https://github.com/wojtpl2)) -* [#927](https://github.com/dotnet/BenchmarkDotNet/pull/927) Improve Disassembly exporters and add PrettyGithubMarkdownDiffDisassemblyExporter (by [@wojtpl2](https://github.com/wojtpl2)) -* [#936](https://github.com/dotnet/BenchmarkDotNet/issues/936) Producing the asm diff reports on demand -* [#937](https://github.com/dotnet/BenchmarkDotNet/pull/937) Producing the asm diff reports on demand - fix for #936 (by [@wojtpl2](https://github.com/wojtpl2)) -* [1903a1](https://github.com/dotnet/BenchmarkDotNet/commit/1903a1bd96d207ed51611d1dc546920f5bfb0d86) Improve Disassembly exporters and add PrettyGithubMarkdownDiffDisassemblyExpo... (by [@wojtpl2](https://github.com/wojtpl2)) -* [dd103b](https://github.com/dotnet/BenchmarkDotNet/commit/dd103b60a4af0d3b9e7efb523c0923e7cbd8b62d) Producing the asm diff reports on demand - fixes #936 (#937) (by [@wojtpl2](https://github.com/wojtpl2)) - -### Improved LINQPad support - -If you run BenchmarkDotNet v0.11.2+ in LINQPad, your logs will be colored and monospaced: - -![linqpad](https://user-images.githubusercontent.com/2259237/47839043-4ec92880-ddc2-11e8-838d-960f6d5449e5.png) - -* [#447](https://github.com/dotnet/BenchmarkDotNet/issues/447) Implement ColoredLogger for LinqPad -* [#903](https://github.com/dotnet/BenchmarkDotNet/pull/903) Add LINQPad logging (by [@bgrainger](https://github.com/bgrainger)) -* [#915](https://github.com/dotnet/BenchmarkDotNet/pull/915) Use a monospaced font for LINQPad logging output (by [@bgrainger](https://github.com/bgrainger)) -* [c3b609](https://github.com/dotnet/BenchmarkDotNet/commit/c3b6095b933b132c1773ced3af126f282465b980) Add LINQPad logging (#903) (by [@bgrainger](https://github.com/bgrainger)) -* [10fdd0](https://github.com/dotnet/BenchmarkDotNet/commit/10fdd0998b46c4358f6fa38aacc21e57a7730724) Use a monospaced font for LINQPad logging output. (#915) (by [@bgrainger](https://github.com/bgrainger)) - -### Better CPU brand strings - -We did a lot of changes which improve the presentation form of the CPU brand string. -Here is an example of such string in the previous version of BenchmarkDotNet: - -```log -AMD Ryzen 7 2700X Eight-Core Processor (Max: 4.10GHz), 1 CPU, 16 logical and 8 physical cores -``` - -Now it becomes: - -```log -AMD Ryzen 7 2700X 4.10GHz, 1 CPU, 16 logical and 8 physical cores -``` - -As you can see, "Eight-Core Processor" was removed (because we already have "8 physical cores"); - "(Max: 4.10GHz)" was replaced by 4.10GHz (because the original CPU brand string doesn't contain the nominal frequency). - -* [#859](https://github.com/dotnet/BenchmarkDotNet/issues/859) Strange max frequency values on Windows (assignee: [@Rizzen](https://github.com/Rizzen)) -* [#909](https://github.com/dotnet/BenchmarkDotNet/issues/909) Improve CPU Brand Strings without frequency -* [#860](https://github.com/dotnet/BenchmarkDotNet/pull/860) Fix strange CPU Frequency values (by [@Rizzen](https://github.com/Rizzen)) -* [#910](https://github.com/dotnet/BenchmarkDotNet/pull/910) Simplify AMD Ryzen CPU brand info (by [@lahma](https://github.com/lahma)) -* [a78b38](https://github.com/dotnet/BenchmarkDotNet/commit/a78b38b0e89d04ad3fe8934162c7adb42f81eabe) Fix strange CPU Frequency values (#860) (by [@Rizzen](https://github.com/Rizzen)) -* [5df1e6](https://github.com/dotnet/BenchmarkDotNet/commit/5df1e6434b791eb5da6f6ef42505fc6a94ebd008) Simplify AMD Ryzen CPU brand info (#910) (by [@lahma](https://github.com/lahma)) - ---- - -## Attributes - -### Async GlobalSetup and GlobalCleanup - -Now GlobalSetup and GlobalCleanup methods can be async. - -See also: docs.setup-and-cleanup - -* [#521](https://github.com/dotnet/BenchmarkDotNet/issues/521) Support async Setup/Cleanup -* [#892](https://github.com/dotnet/BenchmarkDotNet/pull/892) Added support for async GlobalSetup. (by [@dlemstra](https://github.com/dlemstra)) -* [#923](https://github.com/dotnet/BenchmarkDotNet/pull/923) async GlobalCleanup support (by [@dlemstra](https://github.com/dlemstra)) -* [#926](https://github.com/dotnet/BenchmarkDotNet/pull/926) Added support for async GlobalCleanup. (by [@dlemstra](https://github.com/dlemstra)) -* [e0f7a6](https://github.com/dotnet/BenchmarkDotNet/commit/e0f7a67681860ead87cef76fa0db349460b34eb0) Added support for async GlobalSetup. (#892) (by [@dlemstra](https://github.com/dlemstra)) -* [a971a4](https://github.com/dotnet/BenchmarkDotNet/commit/a971a435ce6e6ca25d246e5e2cd56c5b2cf4739d) async GlobalCleanup support (#923) (by [@dlemstra](https://github.com/dlemstra)) -* [e4c7b8](https://github.com/dotnet/BenchmarkDotNet/commit/e4c7b852e5593bb280881e28ece51d26687c5ba9) Added support for async GlobalCleanup. (#926), fixes #521 (by [@dlemstra](https://github.com/dlemstra)) - -### Introduced ParamsAllValues - -If you want to use all possible values of an `enum` or another type with a small number of values, you can use the [`[ParamsAllValues]`](xref:BenchmarkDotNet.Attributes.ParamsAllValuesAttribute) attribute, instead of listing all the values by hand. The types supported by the attribute are: - -* `bool` -* any `enum` that is not marked with `[Flags]` -* `Nullable`, where `T` is an enum or boolean - -An example: - -```cs -public class IntroParamsAllValues -{ - public enum CustomEnum - { - A, - BB, - CCC - } - - [ParamsAllValues] - public CustomEnum E { get; set; } - - [ParamsAllValues] - public bool? B { get; set; } - - [Benchmark] - public void Benchmark() - { - Thread.Sleep( - E.ToString().Length * 100 + - (B == true ? 20 : B == false ? 10 : 0)); - } -} -``` - -Output: - -```markdown - Method | E | B | Mean | Error | ----------- |---- |------ |---------:|------:| - Benchmark | A | ? | 101.9 ms | NA | - Benchmark | A | False | 111.9 ms | NA | - Benchmark | A | True | 122.3 ms | NA | - Benchmark | BB | ? | 201.5 ms | NA | - Benchmark | BB | False | 211.8 ms | NA | - Benchmark | BB | True | 221.4 ms | NA | - Benchmark | CCC | ? | 301.8 ms | NA | - Benchmark | CCC | False | 312.3 ms | NA | - Benchmark | CCC | True | 322.2 ms | NA | - -// * Legends * - E : Value of the 'E' parameter - B : Value of the 'B' parameter -``` - -* [#658](https://github.com/dotnet/BenchmarkDotNet/issues/658) [Params] for enums should include all values by default -* [#908](https://github.com/dotnet/BenchmarkDotNet/pull/908) Added [ParamsAllValues] (by [@gsomix](https://github.com/gsomix)) -* [922dff](https://github.com/dotnet/BenchmarkDotNet/commit/922dfff62d6cf6fd808865e705a09eee63690a2e) Added [ParamsAllValues] (#908), fixes #658 (by [@gsomix](https://github.com/gsomix)) -* [846d08](https://github.com/dotnet/BenchmarkDotNet/commit/846d0863b6456d3e1e6ccab06d8e61c5cd064194) ParamsAllValuesValidator fixes (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) - -### Selecting Baseline across Methods and Jobs - -Now it's possible to mark a method and a job as baselines at the same time: - -```cs -public class TheBaselines -{ - [Benchmark(Baseline = true)] - public void Sleep100ms() => Thread.Sleep(TimeSpan.FromMilliseconds(100)); - - [Benchmark] - public void Sleep50ms() => Thread.Sleep(TimeSpan.FromMilliseconds(50)); -} - -static void Main(string[] args) - => BenchmarkSwitcher - .FromTypes(new[] { typeof(TheBaselines) }) - .Run(args, - DefaultConfig.Instance - .With(Job.Core.AsBaseline()) - .With(Job.Clr.WithId("CLR 4.7.2"))); -``` - -* [#880](https://github.com/dotnet/BenchmarkDotNet/issues/880) Select Baseline across Methods and Jobs (assignee: [@AndreyAkinshin](https://github.com/AndreyAkinshin)) -* [21a007](https://github.com/dotnet/BenchmarkDotNet/commit/21a0073cc8b486f41b2e84deafacd00a1303013a) Support method-job baseline pairs, fixes #880 (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) - ---- - -## Statistics - -### Statistical tests - -In this release, statistical testing was significantly improved. -Now it's possible to compare all benchmarks against baseline with the help - of Welch's t-test or Mann–Whitney U test. - -An example: - -```cs -[StatisticalTestColumn( - StatisticalTestKind.Welch, ThresholdUnit.Microseconds, 1, true)] -[StatisticalTestColumn( - StatisticalTestKind.MannWhitney, ThresholdUnit.Microseconds, 1, true)] -[StatisticalTestColumn( - StatisticalTestKind.Welch, ThresholdUnit.Ratio, 0.03, true)] -[StatisticalTestColumn( - StatisticalTestKind.MannWhitney, ThresholdUnit.Ratio, 0.03, true)] -[SimpleJob(warmupCount: 0, targetCount: 5)] -public class IntroStatisticalTesting -{ - [Benchmark] public void Sleep50() => Thread.Sleep(50); - [Benchmark] public void Sleep97() => Thread.Sleep(97); - [Benchmark] public void Sleep99() => Thread.Sleep(99); - [Benchmark(Baseline = true)] public void Sleep100() => Thread.Sleep(100); - [Benchmark] public void Sleep101() => Thread.Sleep(101); - [Benchmark] public void Sleep103() => Thread.Sleep(103); - [Benchmark] public void Sleep150() => Thread.Sleep(150); -} -``` - -Output: - -| Method | Mean | Error | StdDev | Ratio | Welch(1us)/p-values | Welch(3%)/p-values | MannWhitney(1us)/p-values | MannWhitney(3%)/p-values | -|--------- |----------:|----------:|----------:|------:|---------------------- |---------------------- |-------------------------- |------------------------- | -| Sleep50 | 53.13 ms | 0.5901 ms | 0.1532 ms | 0.51 | Faster: 1.0000/0.0000 | Faster: 1.0000/0.0000 | Faster: 1.0000/0.0040 | Faster: 1.0000/0.0040 | -| Sleep97 | 100.07 ms | 0.9093 ms | 0.2361 ms | 0.97 | Faster: 1.0000/0.0000 | Same: 1.0000/0.1290 | Faster: 1.0000/0.0040 | Same: 1.0000/0.1111 | -| Sleep99 | 102.23 ms | 2.4462 ms | 0.6353 ms | 0.99 | Faster: 0.9928/0.0072 | Same: 1.0000/0.9994 | Faster: 0.9960/0.0079 | Same: 1.0000/1.0000 | -| Sleep100 | 103.34 ms | 0.8180 ms | 0.2124 ms | 1.00 | Base: 0.5029/0.5029 | Base: 1.0000/1.0000 | Base: 0.7262/0.7262 | Base: 1.0000/1.0000 | -| Sleep101 | 103.73 ms | 2.1591 ms | 0.5607 ms | 1.00 | Same: 0.1041/0.8969 | Same: 0.9999/1.0000 | Same: 0.1111/0.9246 | Same: 1.0000/1.0000 | -| Sleep103 | 106.21 ms | 1.2511 ms | 0.3249 ms | 1.03 | Slower: 0.0000/1.0000 | Same: 0.9447/1.0000 | Slower: 0.0040/1.0000 | Same: 0.9246/1.0000 | -| Sleep150 | 153.16 ms | 3.4929 ms | 0.9071 ms | 1.48 | Slower: 0.0000/1.0000 | Slower: 0.0000/1.0000 | Slower: 0.0040/1.0000 | Slower: 0.0040/1.0000 | - -```log -// * Legends * - Mean : Arithmetic mean of all measurements - Error : Half of 99.9% confidence interval - StdDev : Standard deviation of all measurements - Ratio : Mean of the ratio distribution ([Current]/[Baseline]) - Welch(1us)/p-values : Welch-based TOST equivalence test with 1 us threshold. Format: 'Result: p-value(Slower)|p-value(Faster)' - Welch(3%)/p-values : Welch-based TOST equivalence test with 3% threshold. Format: 'Result: p-value(Slower)|p-value(Faster)' - MannWhitney(1us)/p-values : MannWhitney-based TOST equivalence test with 1 us threshold. Format: 'Result: p-value(Slower)|p-value(Faster)' - MannWhitney(3%)/p-values : MannWhitney-based TOST equivalence test with 3% threshold. Format: 'Result: p-value(Slower)|p-value(Faster)' - 1 ms : 1 Millisecond (0.001 sec) -``` - -The statistical testing is a work-in-progress feature. -In future versions of BenchmarkDotNet, we are planning to improve API, fill missed docs, and introduce more parameters for customization. - -See also: @BenchmarkDotNet.Samples.IntroStatisticalTesting - -* [60eca0](https://github.com/dotnet/BenchmarkDotNet/commit/60eca005326970202a33891e5aecd2ef6b7e4cd0) Threshold API for WelchTTest; Improve Student accuracy for small n (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) -* [05cc8d](https://github.com/dotnet/BenchmarkDotNet/commit/05cc8d15ef88e382bbb1827d766d7275c3e42abd) Statistical testing improvements (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) - -### ZeroMeasurementAnalyser - -When you have an empty benchmark like this - -```cs -[Benchmark] -public void Empty() { } -``` - -The expected duration of this method is zero. -However, you can get the mean value like `0.0023ns` because of the natural noise. -It's a pretty confusing result for many developers. -Since v0.11.2, we have `ZeroMeasurementAnalyser` which warn you about such methods. -By default, BenchmarkDotNet automatically evaluate overhead. -In this case, `ZeroMeasurementAnalyser` runs Welch's t-test and compare actual and overhead measurements. -If the overhead evaluation is disabled, it runs one-sample Student's t-test against a half of CPU cycle. - -* [#906](https://github.com/dotnet/BenchmarkDotNet/pull/906) Zero measurement analyser (by [@Rizzen](https://github.com/Rizzen)) -* [48d193](https://github.com/dotnet/BenchmarkDotNet/commit/48d193e30c780eb43e65b21f892c48db5dab6f6b) Zero measurement analyser (#906) (by [@Rizzen](https://github.com/Rizzen)) - -### RatioColumn - -The `Ratio` column was formerly known as `Scaled`. -The old title was a source of misunderstanding and confusion because - many developers interpreted it as the ratio of means (e.g., `50.46`/`100.39` for `Time50`). -The ratio of distribution means and the mean of the ratio distribution are pretty close to each other in most cases, - but they are not equal. - -See also: - @BenchmarkDotNet.Samples.IntroBenchmarkBaseline, - @BenchmarkDotNet.Samples.IntroRatioSD, - @docs.baselines. - -* [4e64c9](https://github.com/dotnet/BenchmarkDotNet/commit/4e64c94cfe7b49bbdc06aabb6ee1f262bd370862) Ratio/RatioSD columns (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) - diff --git a/docs/_changelog/header/v0.11.3.md b/docs/_changelog/header/v0.11.3.md deleted file mode 100644 index f3be5d81af..0000000000 --- a/docs/_changelog/header/v0.11.3.md +++ /dev/null @@ -1,99 +0,0 @@ - - -## Highlights - -This release is focused mainly on bug fixes that were affecting user experience. But don't worry, we have some new features too! - -* **Diagnosers** - * ConcurrencyVisualizerProfiler (allows profiling benchmarks on Windows and exporting the data to a trace file which can be opened with Concurrency Visualizer) -* **Command-line:** - * `--stopOnFirstError`: Stops the benchmarks execution on first error. [#947](https://github.com/dotnet/BenchmarkDotNet/pull/947) - * `--statisticalTest`: Performs a Mann–Whitney Statistical Test for identifying regressions and improvements. [#960](https://github.com/dotnet/BenchmarkDotNet/pull/960) -* **Bug fixes:** - * Dry mode doesn't work because of the ZeroMeasurementHelper [#943](https://github.com/dotnet/BenchmarkDotNet/issues/943) - * MannWhitneyTest fails when comparing statistics of different sample size [#948](https://github.com/dotnet/BenchmarkDotNet/issues/948) and [#950](https://github.com/dotnet/BenchmarkDotNet/issues/950) - * Improve the dynamic loading of Diagnostics package [#955](https://github.com/dotnet/BenchmarkDotNet/issues/955) - * BenchmarkRunner.RunUrl throws NRE when Config is not provided [#961](https://github.com/dotnet/BenchmarkDotNet/issues/961) - * Don't require the users to do manual installation of TraceEvent when using Diagnostics package [#962](https://github.com/dotnet/BenchmarkDotNet/issues/962) - * Stop benchmark after closing application + Flush log after stopping benchmark [#963](https://github.com/dotnet/BenchmarkDotNet/issues/963) - ---- - -## Diagnosers - -### ConcurrencyVisualizerProfiler - -`ConcurrencyVisualizerProfiler` allows to profile the benchmarked .NET code on Windows and exports the data to a CVTrace file which can be opened with [Concurrency Visualizer](https://learn.microsoft.com/visualstudio/profiling/concurrency-visualizer). - -`ConcurrencyVisualizerProfiler` uses `EtwProfiler` to get a `.etl` file which still can be opened with [PerfView](https://github.com/Microsoft/perfview) or [Windows Performance Analyzer](https://learn.microsoft.com/windows-hardware/test/wpt/windows-performance-analyzer). The difference is that it also enables all Task and Thread related ETW Providers and exports a simple `xml` which can be opened with Visual Studio if you install [Concurrency Visualizer plugin](https://marketplace.visualstudio.com/items?itemName=Diagnostics.ConcurrencyVisualizer2017) - -![open trace](https://user-images.githubusercontent.com/6011991/48638184-2b13fe00-e9d0-11e8-8a94-0e951e4606ae.png) - -![Utilization](https://user-images.githubusercontent.com/6011991/48638108-f6a04200-e9cf-11e8-8f7c-feda0a99138c.png) - -![Threads](https://user-images.githubusercontent.com/6011991/48638120-fb64f600-e9cf-11e8-9e2d-53b615bfe9a8.png) - - -* [#964](https://github.com/dotnet/BenchmarkDotNet/issues/964) Concurrency Visualizer Profiler Diagnoser (by [@adamsitnik](https://github.com/adamsitnik)) -* [dfb3c89](https://github.com/dotnet/BenchmarkDotNet/commit/dfb3c8912505799a76b0eb5ae0c082bb44599fa7) ConcurrencyVisualizerProfiler diagnoser! (by [@adamsitnik](https://github.com/adamsitnik)) - ---- - -## Command-line - -In this release, we have some new command-line arguments! - -### `--stopOnFirstError`: Stops the benchmarks execution on first error - -When provided, BenchmarkDotNet is going to stop the benchmarks execution on first error. - -* [#947](https://github.com/dotnet/BenchmarkDotNet/pull/947) Add option to stop running when the first benchmark fails (by [@wojtpl2](https://github.com/wojtpl2)) - -### `--statisticalTest`: Statistical Test - -To perform a Mann–Whitney U Test and display the results in a dedicated column you need to provide the Threshold via -`--statisticalTest`. Examples: 5%, 10ms, 100ns, 1s. - -Example: run Mann–Whitney U test with relative ratio of 1% for all benchmarks for .NET 4.6 (base), .NET Core 2.0 and .NET Core 2.1. - -```cs -class Program -{ - static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); -} - -public class MySample -{ - [Benchmark] - public void Sleep() - { -#if NETFRAMEWORK - Thread.Sleep(50); -#elif NETCOREAPP2_0 - Thread.Sleep(45); -#elif NETCOREAPP2_1 - Thread.Sleep(55); -#endif - } - - [Benchmark] - public void Same() => Thread.Sleep(50); -} -``` - -```log -dotnet run -c Release -f netcoreapp2.1 --filter * --runtimes net46 netcoreapp2.0 netcoreapp2.1 --statisticalTest 1% -``` - -**Note:** .NET 4.6 will be our baseline because it was provided as first on the runtimes list. - - -| Method | Runtime | Toolchain | Mean | Error | StdDev | Ratio | MannWhitney(1%) | -|------- |-------- |-------------- |---------:|----------:|----------:|------:|---------------- | -| Sleep | Clr | net46 | 50.51 ms | 0.1833 ms | 0.1714 ms | 1.00 | Base | -| Sleep | Core | netcoreapp2.0 | 45.53 ms | 0.1262 ms | 0.1181 ms | 0.90 | Faster | -| Sleep | Core | netcoreapp2.1 | 55.50 ms | 0.1217 ms | 0.1138 ms | 1.10 | Slower | -| | | | | | | | | -| Same | Clr | net46 | 50.47 ms | 0.1795 ms | 0.1679 ms | 1.00 | Base | -| Same | Core | netcoreapp2.0 | 50.55 ms | 0.1873 ms | 0.1752 ms | 1.00 | Same | -| Same | Core | netcoreapp2.1 | 50.55 ms | 0.2162 ms | 0.2022 ms | 1.00 | Same | diff --git a/docs/_changelog/header/v0.11.4.md b/docs/_changelog/header/v0.11.4.md deleted file mode 100644 index 3cf8b79f55..0000000000 --- a/docs/_changelog/header/v0.11.4.md +++ /dev/null @@ -1,114 +0,0 @@ -It's been few months since our last release, but we have been working hard and have some new features for you! - -## Highlights - -* **Features** - * **BenchmarkDotNet as a global tool**: a new global tool which allows you to run benchmarks from given library. - Now you can run benchmarks from the command line via `dotnet benchmark`. - You can find more information about it in the documentation - [#1006](https://github.com/dotnet/BenchmarkDotNet/pull/1006) - [#213](https://github.com/dotnet/BenchmarkDotNet/issues/213) - * **InProcessEmitToolchain**: new, full-featured InProcess toolchain which allows executing benchmarks in the current process - without spawning additional process per benchmark. - It supports `[Arguments]`, `[ArgumentsSource]`, passing the arguments by `out`, `ref` and returning stack-only types like `Span`. - [#919](https://github.com/dotnet/BenchmarkDotNet/issues/919), - [#921](https://github.com/dotnet/BenchmarkDotNet/pull/921) - [#843](https://github.com/dotnet/BenchmarkDotNet/issues/843) - * **ARM support**: BenchmarkDotNet supports now ARM and ARM64. - [#780](https://github.com/dotnet/BenchmarkDotNet/issues/780), - [#979](https://github.com/dotnet/BenchmarkDotNet/pull/979) - [#385](https://github.com/dotnet/BenchmarkDotNet/issues/385) - * **Mono AOT support**: a new toolchain which allows running benchmarks using AOT version of Mono - [#940](https://github.com/dotnet/BenchmarkDotNet/pull/940) - * **NuGet symbol server support**: BenchmarkDotNet publishes now the symbols to NuGet.org symbol server and you can easily debug it. - [#967](https://github.com/dotnet/BenchmarkDotNet/issues/967) - [#968](https://github.com/dotnet/BenchmarkDotNet/pull/968) - * **Experimental support for .NET Core 3.0 WPF benchmarks** - [#1066](https://github.com/dotnet/BenchmarkDotNet/pull/1066) - For a working example please go to [https://github.com/dotMorten/WPFBenchmarkTests](https://github.com/dotMorten/WPFBenchmarkTests) -* **Improvements:** - * CoreRT Toolchain improvements - thanks to help from CoreRT Team we were able to make - the CoreRT Toolchain work with the latest version of CoreRT - [#1001](https://github.com/dotnet/BenchmarkDotNet/pull/1001), - [#1057](https://github.com/dotnet/BenchmarkDotNet/pull/1057) - * Display the number of benchmarks to run: we now display how many benchmarks are going to be executed - before running them and how many remained after running each of them - [#1048](https://github.com/dotnet/BenchmarkDotNet/issues/1048) - * Better list of suggested benchmarks for wrong filter - [#834](https://github.com/dotnet/BenchmarkDotNet/issues/834) - [#957](https://github.com/dotnet/BenchmarkDotNet/pull/957) - * Invalid assembly binding redirects generated by VS were a pain to many of our users, - we have now implemented an approach that tries to work around this issue. - [#895](https://github.com/dotnet/BenchmarkDotNet/issues/895), - [#667](https://github.com/dotnet/BenchmarkDotNet/issues/667), - [#896](https://github.com/dotnet/BenchmarkDotNet/issues/896), - [#942](https://github.com/dotnet/BenchmarkDotNet/issues/942) - * Handling duplicates in IConfig - [#912](https://github.com/dotnet/BenchmarkDotNet/pull/912), - [#938](https://github.com/dotnet/BenchmarkDotNet/issues/938), - [#360](https://github.com/dotnet/BenchmarkDotNet/issues/360), - [#463](https://github.com/dotnet/BenchmarkDotNet/issues/463) - * Disassembly diagnoser should be kept in a separate directory to avoid dependency conflicts - [#1059](https://github.com/dotnet/BenchmarkDotNet/issues/1059) - * Give a warning when the `[Benchmark]` method is static - we now produce an error when users fail into this common issue - [#983](https://github.com/dotnet/BenchmarkDotNet/issues/983) - [#985](https://github.com/dotnet/BenchmarkDotNet/pull/985) - * C# keywords are prohibited as benchmark names - [#849](https://github.com/dotnet/BenchmarkDotNet/issues/849) - * File names should be consistent across all OSes - `<` and `>` are valid on Unix, but not on Windows. - We have unified that and now files produced on Unix and Windows have the same names. - [#981](https://github.com/dotnet/BenchmarkDotNet/issues/981) - * Improve restore, build and publish projects - [#1002](https://github.com/dotnet/BenchmarkDotNet/issues/1002), - [#1013](https://github.com/dotnet/BenchmarkDotNet/pull/1013) - * Make it possible to disable OptimizationsValidator - [#988](https://github.com/dotnet/BenchmarkDotNet/issues/988) - * Sort enum parameters by value instead of name - [#977](https://github.com/dotnet/BenchmarkDotNet/pull/977) - * Detect .NET Core benchmark failures from LINQPad - [#980](https://github.com/dotnet/BenchmarkDotNet/pull/980) - * Improved error logging - [#1008](https://github.com/dotnet/BenchmarkDotNet/pull/1008) - * Improved disassembly diff - [#1022](https://github.com/dotnet/BenchmarkDotNet/pull/1022) - * Using invariant culture for Roslyn Toolchain error messages - [#1042](https://github.com/dotnet/BenchmarkDotNet/pull/1042) - * Use only full names in the auto-generated code to avoid any possible conflicts with user code - [#1007](https://github.com/dotnet/BenchmarkDotNet/issues/1007), - [#1009](https://github.com/dotnet/BenchmarkDotNet/pull/1009) - [#1010](https://github.com/dotnet/BenchmarkDotNet/issues/1010) - * Write the GitHub table format to the console by default - [#1062](https://github.com/dotnet/BenchmarkDotNet/issues/1062) - * Proper cleanup on Ctrl+C/console Window exit - [#1061](https://github.com/dotnet/BenchmarkDotNet/pull/1061) - * Introduce StoppingCriteria - the first step to writing your own heuristic that determines when benchmarking should be stopped - [#984](https://github.com/dotnet/BenchmarkDotNet/pull/984) -* **Breaking changes:** - * .NET Standard 2.0 only - BenchmarkDotNet has a single target now, which should help with some assembly resolving issues. - We had to drop .NET 4.6 support because of that and .NET 4.6.1 is now the oldest supported .NET Framework. - [#1032](https://github.com/dotnet/BenchmarkDotNet/pull/1032) - * CustomCoreClrToolchain has been removed, it's recommended to use CoreRunToolchain instead - [#928](https://github.com/dotnet/BenchmarkDotNet/issues/928) -* **Bug fixes:** - * NRE in `Summary` ctor - [#986](https://github.com/dotnet/BenchmarkDotNet/issues/986) - [#987](https://github.com/dotnet/BenchmarkDotNet/pull/987) - * ArgumentNullException when running benchmarks from published .NET Core app - [#1018](https://github.com/dotnet/BenchmarkDotNet/issues/1018) - * Dry jobs can eat iteration failures - [#1045](https://github.com/dotnet/BenchmarkDotNet/issues/1045) - * NullReferenceException in BenchmarkDotNet.Reports.SummaryTable after iteration failure - [#1046](https://github.com/dotnet/BenchmarkDotNet/issues/1046) - * Running the example throws NullReference - [#1049](https://github.com/dotnet/BenchmarkDotNet/issues/1049) - * Fix race condition in process output reader - [#1051](https://github.com/dotnet/BenchmarkDotNet/issues/1051) - [#1053](https://github.com/dotnet/BenchmarkDotNet/pull/1053) - * Fix a rare but really annoying bug where for some reason we were sometimes setting ForegroundColor - to the same color as BackgroundColor and some parts of the logged output were invisible - [commit](https://github.com/dotnet/BenchmarkDotNet/commit/ea3036810ef60b483d766a097e6f3edfde28a834) - * StopOnFirstError must be respected - [commit](https://github.com/dotnet/BenchmarkDotNet/commit/87d281d7dbf52036819efff52e6661e436648b73) - ---- - diff --git a/docs/_changelog/header/v0.11.5.md b/docs/_changelog/header/v0.11.5.md deleted file mode 100644 index 98e31c0195..0000000000 --- a/docs/_changelog/header/v0.11.5.md +++ /dev/null @@ -1,170 +0,0 @@ -## Highlights - -* **Features and noticeable improvements** - * **Power plan management** - Now BenchmarkDotNet executes all benchmarks with enabled High-Performance power plan (configurable, Windows-only). - You can find some details below. - [#68](https://github.com/dotnet/BenchmarkDotNet/issues/68) - [#952](https://github.com/dotnet/BenchmarkDotNet/pull/952) - * **Better Environment Variables API** - Now we have some additional extension methods which allow defining environment variables in user jobs. - In the previous version, users always had to set an array of environment variables like this: - `job.With(new[] { new EnvironmentVariable("a", "b") })`. - Now it's possible to define an environment variable like - `job.With(new EnvironmentVariable("a", "b"))` or - `job.WithEnvironmentVariable("a", "b")`. - Also, it's possible to clear the list of environment variables via - `job.WithoutEnvironmentVariables()`. - [#1069](https://github.com/dotnet/BenchmarkDotNet/issues/1069) - [#1080](https://github.com/dotnet/BenchmarkDotNet/pull/1080) - * **Better outlier messages** - The previous version of BenchmarkDotNet printed information about detected or removed outliers like this: - "3 outliers were detected". - It was nice, but it didn't provide additional information about these outliers - (users had to read the full log to find the outliers values). - Now BenchmarkDotNet prints additional information about outlier values like this: - "3 outliers were detected (2.50 us..2.70 us)". - [e983cd31](https://github.com/dotnet/BenchmarkDotNet/commit/e983cd3126e64f82fe59bc1bc45d1a870a615e87) - * **Support modern CPU architecture names** - In the environment information section, BenchmarkDotNet prints not only the processor brand string, but also its architecture - (e.g., "Intel Core i7-4770K CPU 3.50GHz (Haswell)"). - However, it failed to recognize some recent processors. - Now it's able to detect the architecture for modern Intel processors correctly - (Kaby Lake, Kaby Lake R, Kaby Lake G, Amber Lake Y, Coffee Lake, Cannon Lake, Whiskey Lake). - [995e053d](https://github.com/dotnet/BenchmarkDotNet/commit/995e053d14a61cdadc417149480f23ebf679bcb7) - * **Introduce BenchmarkDotNet.Annotations** - Currently, BenchmarkDotNet targets .NET Standard 2.0. - It makes some users unhappy because they want to define benchmarks in projects with lower target framework. - We decided to start working on the `BenchmarkDotNet.Annotations` NuGet package which targets .NET Standard 1.0 - and contains classes that users need to define their benchmarks. - However, it's not easy to refactor the full source code base and move all relevant public APIs to this package. - In v0.11.5, we did the first step and moved some of these APIs to `BenchmarkDotNet.Annotations`. - We want to continue moving classes to this package and get full-featured annotation package in the future. - [#1084](https://github.com/dotnet/BenchmarkDotNet/pull/1084) - [#1096](https://github.com/dotnet/BenchmarkDotNet/pull/1096) - * **Use InProcessEmitToolchain by default in InProcess benchmarks** - In BenchmarkDotNet 0.11.4, we introduced `InProcessEmitToolchain`. - It's a new, full-featured InProcess toolchain which allows executing benchmarks in the current process - without spawning additional process per benchmark. - It supports `[Arguments]`, `[ArgumentsSource]`, passing the arguments by `out`, `ref` and returning stack-only types like `Span`. - However, in v0.11.4, it can be activated only if `InProcessEmitToolchain` is declared explicitly. - Now it's enabled by default when `[InProcessAttribute]` is used. - [#1093](https://github.com/dotnet/BenchmarkDotNet/pull/1093) - * **Introduce an option which prevents overwriting results** - Currently, BenchmarkDotNet overwrites results each time when the benchmarks are executed. - It allows avoiding tons of obsolete files in the `BenchmarkDotNet.Artifacts` folder. - However, the behavior doesn't fit all use cases: - sometimes users want to keep results for old benchmark runs. - Now we have a special option for it. - The option can be activated via `--noOverwrite` console line argument or - `DontOverwriteResults` extension method for `IConfig` - [#1074](https://github.com/dotnet/BenchmarkDotNet/issues/1074) - [#1083](https://github.com/dotnet/BenchmarkDotNet/pull/1083) -* **Other improvements and bug fixes** - * **Diagnostics and validation** - * **Better benchmark declaration error processing** - In the previous version, BenchmarkDotNet threw an exception when some benchmark methods had an invalid declaration - (e.g., invalid signature or invalid access modifiers). - Now it prints a nice error message without ugly stack traces. - [#1107](https://github.com/dotnet/BenchmarkDotNet/issues/1107) - * **Better error message for users who want to debug benchmarks** - [#1073](https://github.com/dotnet/BenchmarkDotNet/issues/1073) - * **Don't show the same validation error multiple times** - Now each error will be printed only once. - [#1079](https://github.com/dotnet/BenchmarkDotNet/issues/1079) - * **Restrict MemoryDiagnoserAttribute usage to class** - Now it's impossible to accidentally mark a method with this attribute. - [#1119](https://github.com/dotnet/BenchmarkDotNet/issues/1119) - [#1122](https://github.com/dotnet/BenchmarkDotNet/pull/1122) - * **Export** - * **Better indentation in disassembly listings** - Now DissassemblyDiagnoser correctly process source code which contains tab as the indentation symbol - [#1110](https://github.com/dotnet/BenchmarkDotNet/issues/1110) - * **Fix incorrect indentation for StackOverflow exporter** - Previously, StackOverflow exporter doesn't have a proper indent for job runtimes in the environment information. - Now it's fixed. - [#826](https://github.com/dotnet/BenchmarkDotNet/issues/826) - [#1104](https://github.com/dotnet/BenchmarkDotNet/pull/1104) - * **Fix StackOverflowException in XmlExporter.Full** - [#1086](https://github.com/dotnet/BenchmarkDotNet/issues/1086) - [#1090](https://github.com/dotnet/BenchmarkDotNet/pull/1090) - * **Shortify MemoryDiagnoser column titles** - Now we use the following column titles: - "Allocated" instead of "Allocated Memory/Op", - "Gen 0" instead of "Gen 0/1k Op". - The full description of each column can be found in the legend section below the summary table. - [#1081](https://github.com/dotnet/BenchmarkDotNet/pull/1081) - * **Benchmark generation and execution** - * **Fixed broken Orderers** - The previous version has a nasty bug with custom orderers. - Now it's fixed. - [#1070](https://github.com/dotnet/BenchmarkDotNet/issues/1070) - [#1109](https://github.com/dotnet/BenchmarkDotNet/issues/1109) - * **Better overhead evaluation** - In the previous version, BenchmarkDotNet evaluated the benchmark overhead as a mean value of all overhead iteration. - It was fine in most cases, but in some cases, the mean value can be spoiled by outliers. - Now BenchmarkDotNet uses the median value. - [#1116](https://github.com/dotnet/BenchmarkDotNet/issues/1116) - * **Respect CopyLocalLockFileAssemblies** - Now BenchmarkDotNet respect the `CopyLocalLockFileAssemblies` value and copies it - to the generated benchmark project. - [#1068](https://github.com/dotnet/BenchmarkDotNet/issues/1068) - [#1108](https://github.com/dotnet/BenchmarkDotNet/pull/1108) - * **Disable CodeAnalysisRuleSet for generated benchmarks** - Previously, generated benchmarks may fail if the `CodeAnalysisRuleSet` is defined in `Directory.Build.Props`. - [#1082](https://github.com/dotnet/BenchmarkDotNet/pull/1082) - * **Supported undefined enum values** - [#1020](https://github.com/dotnet/BenchmarkDotNet/issues/1020) - [#1071](https://github.com/dotnet/BenchmarkDotNet/issues/1071) - * **Other minor improvements and bug fixes** - -## Power plans - -In [#952](https://github.com/dotnet/BenchmarkDotNet/pull/952), power plan management was implemented. -It resolves a pretty old issue [#68](https://github.com/dotnet/BenchmarkDotNet/issues/68) which was created more than three years ago. -Now BenchmarkDotNet forces OS to execute a benchmark on the High-Performance power plan. -You can disable this feature by modifying PowerPlanMode property. -Here is an example where we are playing with this value: - -```cs -[Config(typeof(Config))] -public class IntroPowerPlan -{ - private class Config : ManualConfig - { - public Config() - { - Add(Job.MediumRun.WithPowerPlan(PowerPlan.HighPerformance)); - Add(Job.MediumRun.WithPowerPlan(PowerPlan.UserPowerPlan)); - } - } - - [Benchmark] - public int IterationTest() - { - int j = 0; - for (int i = 0; i < short.MaxValue; ++i) - j = i; - return j; - } - - [Benchmark] - public int SplitJoin() - => string.Join(",", new string[1000]).Split(',').Length; -} -``` - -And here is an example of the summary table on plugged-off laptop: - -```md - Method | PowerPlan | Mean | Error | StdDev | --------------- |---------------- |---------:|----------:|----------:| - IterationTest | HighPerformance | 40.80 us | 0.4168 us | 0.6109 us | - SplitJoin | HighPerformance | 13.24 us | 0.2514 us | 0.3763 us | - IterationTest | UserPowerPlan | 79.72 us | 2.5623 us | 3.8352 us | - SplitJoin | UserPowerPlan | 24.54 us | 2.1062 us | 3.1525 us | -``` - -As you can see, the power plan produces a noticeable effect on the benchmark results. - -This feature is available on Windows only. \ No newline at end of file diff --git a/docs/_changelog/header/v0.12.0.md b/docs/_changelog/header/v0.12.0.md deleted file mode 100644 index 0e6a00582f..0000000000 --- a/docs/_changelog/header/v0.12.0.md +++ /dev/null @@ -1,469 +0,0 @@ -It's been several months since our last release, but we have been working hard and have some new features for you! - -## Highlights - -* **Features and major improvements** - * **Advanced multiple target frameworks support** - Now BenchmarkDotNet supports .NET Framework 4.8, .NET Core 3.1, and .NET Core 5.0. - Also, we reworked our API that allows targeting several runtimes from the same config: - the new API is more consistent, flexible, and powerful. - For example, if you want to execute your benchmarking using .NET Framework 4.8 and .NET Core 3.1, - you can use the `SimpleJob(RuntimeMoniker.Net48)`, `[SimpleJob(RuntimeMoniker.NetCoreApp31)]` attributes or - `Job.Default.With(ClrRuntime.Net48)`, `Job.Default.With(CoreRuntime.Core31)` jobs in a manual config. - You can find more details below. - [#1188](https://github.com/dotnet/BenchmarkDotNet/pull/1188), - [#1186](https://github.com/dotnet/BenchmarkDotNet/issues/1186), - [#1236](https://github.com/dotnet/BenchmarkDotNet/issues/1236) - * **Official templates for BenchmarkDotNet-based projects** - With the help of the [BenchmarkDotNet.Templates](https://www.nuget.org/packages/BenchmarkDotNet.Templates/) NuGet package, - you can easily create new projects from the command line via `dotnet new benchmark`. - This command has a lot of useful options, so you can customize your new project as you want. - [#1044](https://github.com/dotnet/BenchmarkDotNet/pull/1044) - * **New NativeMemoryProfiler** - `NativeMemoryProfiler` measure the native memory traffic and adds the extra columns `Allocated native memory` and `Native memory leak` to the summary table. - Internally, it uses `EtwProfiler` to profile the code using ETW. - [#457](https://github.com/dotnet/BenchmarkDotNet/issues/457), - [#1131](https://github.com/dotnet/BenchmarkDotNet/pull/1131), - [#1208](https://github.com/dotnet/BenchmarkDotNet/pull/1208), - [#1214](https://github.com/dotnet/BenchmarkDotNet/pull/1214), - [#1218](https://github.com/dotnet/BenchmarkDotNet/pull/1218), - [#1219](https://github.com/dotnet/BenchmarkDotNet/pull/1219) - * **New ThreadingDiagnoser** - `ThreadingDiagnoser` also adds two extra columns to the summary table: - `Completed Work Items` (the number of work items that have been processed in ThreadPool per single operation) and - `Lock Contentions` (the number of times there *was contention* upon trying to take a Monitor's lock per single operation). - Internally, it uses [new APIs](https://github.com/dotnet/corefx/issues/35500) exposed in .NET Core 3.0. - [#1154](https://github.com/dotnet/BenchmarkDotNet/issues/1154), - [#1227](https://github.com/dotnet/BenchmarkDotNet/pull/1227) - * **Improved MemoryDiagnoser** - Now `MemoryDiagnoser` includes memory allocated by *all threads* that were live during benchmark execution: a new GC API was exposed in .NET Core 3.0 preview6+. - It allows to get the number of allocated bytes for all threads. - [#1155](https://github.com/dotnet/BenchmarkDotNet/pull/1155), - [#1153](https://github.com/dotnet/BenchmarkDotNet/issues/1153), - [#723](https://github.com/dotnet/BenchmarkDotNet/issues/723) - * **LINQPad 6 support** - Now both LINQPad 5 and LINQPad 6 are supported! - [#1241](https://github.com/dotnet/BenchmarkDotNet/issues/1241), - [#1245](https://github.com/dotnet/BenchmarkDotNet/pull/1245) - * **Fast documentation search** - We continue to improve the usability of our documentation. - In this release, we improved the search experience in the documentation: now it works almost instantly with the help of [Algolia](https://www.algolia.com/) engine! - [#1148](https://github.com/dotnet/BenchmarkDotNet/pull/1148), - [#1158](https://github.com/dotnet/BenchmarkDotNet/issues/1158) -* **Minor summary and exporter improvements** - * **Improved presentation of the current architecture in the environment information** - In the previous version of BenchmarkDotNet, the reports always contained "64bit" or "32bit" which did not tell if it was ARM or not. - Now it prints the full architecture name (`x64`, `x86`, `ARM`, or `ARM64`). - For example, instead of `.NET Framework 4.8 (4.8.3815.0), 64bit RyuJIT` you will get `.NET Framework 4.8 (4.8.3815.0), X64 RyuJIT` or `.NET Framework 4.8 (4.8.3815.0), ARM64 RyuJIT`. - [#1213](https://github.com/dotnet/BenchmarkDotNet/pull/1213) - * **Simplified reports for Full .NET Framework version** - Previous version: `.NET Framework 4.7.2 (CLR 4.0.30319.42000), 64bit RyuJIT-v4.7.3324.0`. - Current version: `.NET Framework 4.7.2 (4.7.3362.0), 64bit RyuJIT`. - [#1114](https://github.com/dotnet/BenchmarkDotNet/pull/1114), - [#1111](https://github.com/dotnet/BenchmarkDotNet/issues/1111) - * **More reliable CPU info on Windows** - We added a workaround to for a bug in wmic that uses `\r\r\n` as a line separator. - [#1144](https://github.com/dotnet/BenchmarkDotNet/issues/1144), - [#1145](https://github.com/dotnet/BenchmarkDotNet/pull/1145) - * **Better naming for generated plots** - When `[RPlotExporter]` is used, BenchmarkDotNet generates a lot of useful plots in the `BenchmarkDotNet.Artifacts` folder. - The naming of the plot files was improved: benchmarks without `Params` doesn't include a double dash (`--`) in their names anymore. - [1183](https://github.com/dotnet/BenchmarkDotNet/issues/1183), - [1212](https://github.com/dotnet/BenchmarkDotNet/pull/1212) - * **Better density plot precision** - The previous version of BenchmarkDotNet used the rule-of-thumb bandwidth selector in `RPlotExporter` density plots. - It was fine for unimodal distributions, but sometimes it produced misleading plots for multimodal distributions. - Now, RPlotExporter uses the Sheather&Jones bandwidth selector that significantly improves the presentation of the density plots for complex distributions. - [58fde64](https://github.com/dotnet/BenchmarkDotNet/commit/58fde64c809ceadb3fca9d677a7cec83071b833f) - * **Better alignment in `HtmlExporter`** - Now BenchmarkDotNet aligns the content exported by `HtmlExporter` to the right. - [#1189](https://github.com/dotnet/BenchmarkDotNet/pull/1189) - [dfa074](https://github.com/dotnet/BenchmarkDotNet/commit/dfa074b024cfa8bbe4fe175d3d65a3d9f85127ff) - * **Better precision calculation in SummaryTable** - [4e9eb43](https://github.com/dotnet/BenchmarkDotNet/commit/4e9eb4335eee05a95a3766f2c81ae260508021af) - * **Better summary analysis** - BenchmarkDotNet warns the user when benchmark baseline value is too close to zero and the columns derived from BaselineCustomColumn cannot be computed. - [#1161](https://github.com/dotnet/BenchmarkDotNet/pull/1161), - [#600](https://github.com/dotnet/BenchmarkDotNet/issues/600) - * **Make log file datetime format 24-hour** - [#1149](https://github.com/dotnet/BenchmarkDotNet/issues/1149) - * **Improve AskUser prompt message** - The error messages will surround `*` by quotes on Linux and macOS. - [#1147](https://github.com/dotnet/BenchmarkDotNet/issues/1147) -* **Minor API improvements** - * **ED-PELT algorithm for changepoint detection is now available** - You can find details in [this blog post](https://aakinshin.net/posts/edpelt/). - [f89091](https://github.com/dotnet/BenchmarkDotNet/commit/f89091a2a9c1a4058dd8e32d5aa01271910dd7dc) - * **Improved OutlierMode API** - BenchmarkDotNet performs measurement postprocessing that may remove some of the outlier values (it can be useful to remove upper outliers that we get because of the natural CPU noise). - In the previous version, naming for the `OutlierMode` values was pretty confusing: `None/OnlyUpper/OnlyLower/All`. - Now, these values were renamed to `DontRemove/RemoveUpper/RemoveLower/RemoveAll`. - For example, if you want to remove all the outliers, you can annotate your benchmark with the `[Outliers(OutlierMode.RemoveAll)]` attribute. - The old names still exist (to make sure that the changes are backward compatible), but they are marked as obsolete, and they will be removed in the future versions of the library. - [#1199](https://github.com/dotnet/BenchmarkDotNet/pull/1199), - [0e4b8e](https://github.com/dotnet/BenchmarkDotNet/commit/0e4b8e69d10e73a83fce3ce3980497ee7798bc87) - * **Add the possibility to pass `Config` to `BenchmarkSwitcher.RunAll` and `RunAllJoined`** - [#1194](https://github.com/dotnet/BenchmarkDotNet/issues/1194), - [ae23bd](https://github.com/dotnet/BenchmarkDotNet/commit/ae23bddebe49cd66a9627790c073b7bc45ccbf5c) - * **Improved command line experience** - When user uses `--packages $path`, the `$path` will be sent to the dotnet build command as well. - [1187](https://github.com/dotnet/BenchmarkDotNet/issues/1187) - * **Extend the list of supported power plans.** - Now it supports "ultimate", "balanced", and "power saver" plans. - [#1132](https://github.com/dotnet/BenchmarkDotNet/issues/1132), - [#1139](https://github.com/dotnet/BenchmarkDotNet/pull/1139) - * **Make it possible to not enforce power plan on Windows.** - [1578c5c](https://github.com/dotnet/BenchmarkDotNet/commit/1578c5c60c3f59f9128f680e35d1db219aa60d8d) - * **`Guid` support in benchmark arguments** - Now you can use `Guid` instances as benchmark arguments. - [04ec20b](https://github.com/dotnet/BenchmarkDotNet/commit/04ec20b5e0c0a514e8d158684864e4f9934ae8cc) - * **Make `ArgumentsSource` support `IEnumerable` for benchmarks accepting a single argument to mimic `MemberData` behaviour.** - [ec296dc](https://github.com/dotnet/BenchmarkDotNet/commit/ec296dc45de798f7407852d5ab7febe2b457eca4) - * **Make `FullNameProvider` public** - So it can be reused by the `dotnet/performance` repository. - [6d71308](https://github.com/dotnet/BenchmarkDotNet/commit/6d71308f4c1d96bcf8a526bfcc61bb23307b4041) - * **Extend `Summary` with `LogFilePath`** - [#1135](https://github.com/dotnet/BenchmarkDotNet/issues/1135), - [6e6559](https://github.com/dotnet/BenchmarkDotNet/commit/6e6559499652c312369fd223d0cf4747f65512d6) - * **Allow namespace filtering for `InliningDiagnoser`** - [#1106](https://github.com/dotnet/BenchmarkDotNet/issues/1106), - [#1130](https://github.com/dotnet/BenchmarkDotNet/pull/1130) - * **Option to configure `MaxParameterColumnWidth`** - [#1269](https://github.com/dotnet/BenchmarkDotNet/issues/1269), - [4ec888](https://github.com/dotnet/BenchmarkDotNet/commit/4ec88844547507474ccd0303b31f935b3463318c) -* **Other improvements** - * **Misc improvements in the documentation** - [#1175](https://github.com/dotnet/BenchmarkDotNet/pull/1175), - [#1173](https://github.com/dotnet/BenchmarkDotNet/pull/1173), - [#1180](https://github.com/dotnet/BenchmarkDotNet/pull/1180), - [#1203](https://github.com/dotnet/BenchmarkDotNet/pull/1203), - [#1204](https://github.com/dotnet/BenchmarkDotNet/pull/1204), - [#1206](https://github.com/dotnet/BenchmarkDotNet/pull/1206), - [#1209](https://github.com/dotnet/BenchmarkDotNet/pull/1209), - [#1219](https://github.com/dotnet/BenchmarkDotNet/pull/1219), - [#1225](https://github.com/dotnet/BenchmarkDotNet/pull/1225), - [#1279](https://github.com/dotnet/BenchmarkDotNet/pull/1279) - * **Copy `PreserveCompilationContext` MSBuild setting from the project that defines benchmarks** - [#1152](https://github.com/dotnet/BenchmarkDotNet/issues/1152), - [063d1a](https://github.com/dotnet/BenchmarkDotNet/commit/063d1a56152fd5812cb6e9dd5095dc6e647e6938) - * **Add `System.Buffers.ArrayPoolEventSource` to the list of default .NET Providers of `EtwProfiler`** - [#1179](https://github.com/dotnet/BenchmarkDotNet/issues/1179), - [a106b1](https://github.com/dotnet/BenchmarkDotNet/commit/a106b114b1f04fa1024be84a8969f5a168fa1c8b) - * **Consume CoreRT from the new NuGet feed** - Because CoreRT no longer publishes to MyGet. - [#1129](https://github.com/dotnet/BenchmarkDotNet/pull/1129) -* **Breaking changes:** - * The `[ClrJob]`, `[CoreJob]` and `[CoreRtJob]` attributes got obsoleted and replaced by a `[SimpleJob]` which requires the user to provide target framework moniker in an explicit way. - (See the "Advanced multiple target frameworks support" section for details.) - [#1188](https://github.com/dotnet/BenchmarkDotNet/pull/1188), - [#1182](https://github.com/dotnet/BenchmarkDotNet/issues/1182), - [#1115](https://github.com/dotnet/BenchmarkDotNet/issues/1115), - [#1056](https://github.com/dotnet/BenchmarkDotNet/issues/1056), - [#993](https://github.com/dotnet/BenchmarkDotNet/issues/993), - * The old `InProcessToolchain` is now obsolete. It's recommended to use `InProcessEmitToolchain`. If you want to use the old one on purpose, you have to use `InProcessNoEmitToolchain`. - [#1123](https://github.com/dotnet/BenchmarkDotNet/pull/1123) -* **Bug fixes:** - * Invalid arg passing in StreamLogger constructor. The `append` arg was not passed to the `StreamWriter` .ctor. - [#1185](https://github.com/dotnet/BenchmarkDotNet/pull/1185) - * Improve the output path of `.etl` files produced by `EtwProfiler`. `EtwProfiler` was throwing NRE for users who were using `[ClrJob]` and `[CoreJob]` attributes. - [#1156](https://github.com/dotnet/BenchmarkDotNet/issues/1156), - [#1072](https://github.com/dotnet/BenchmarkDotNet/issues/1072) - * Flush custom loggers at the end of benchmark session. - [#1134](https://github.com/dotnet/BenchmarkDotNet/issues/1134) - * Make ids for tag columns unique - when using multiple `TagColumns` only one `TagColumn` was printed in the results. - [#1146](https://github.com/dotnet/BenchmarkDotNet/issues/1146) - -## Advanced multiple target frameworks support - -Now BenchmarkDotNet supports .NET Framework 4.8, .NET Core 3.1, and .NET Core 5.0. -Also, we reworked our API that allows targeting several runtimes from the same config: - the new API is more consistent, flexible, and powerful. -For example, if you want to execute your benchmarking using .NET Framework 4.8 and .NET Core 3.1, - you can use the `SimpleJob(RuntimeMoniker.Net48)`, `[SimpleJob(RuntimeMoniker.NetCoreApp31)]` attributes or - `Job.Default.With(ClrRuntime.Net48)`, `Job.Default.With(CoreRuntime.Core31)` jobs in a manual config. - -Now let's discuss how to use it in detail. -If you want to test multiple frameworks, your project file **MUST target all of them** and you **MUST install the corresponding SDKs**: - -```xml -netcoreapp3.0;netcoreapp2.1;net48 -``` - -If you run your benchmarks without specifying any custom settings, BenchmarkDotNet is going to run the benchmarks **using the same framework as the host process** (it corresponds to `RuntimeMoniker.HostProcess`): - -```cmd -dotnet run -c Release -f netcoreapp2.1 # is going to run the benchmarks using .NET Core 2.1 -dotnet run -c Release -f netcoreapp3.0 # is going to run the benchmarks using .NET Core 3.0 -dotnet run -c Release -f net48 # is going to run the benchmarks using .NET 4.8 -mono $pathToExe # is going to run the benchmarks using Mono from your PATH -``` - -To run the benchmarks for multiple runtimes with a single command from the command line, you need to specify the runtime moniker names via `--runtimes|-r` console argument: - -```cmd -dotnet run -c Release -f netcoreapp2.1 --runtimes netcoreapp2.1 netcoreapp3.0 # is going to run the benchmarks using .NET Core 2.1 and .NET Core 3.0 -dotnet run -c Release -f netcoreapp2.1 --runtimes netcoreapp2.1 net48 # is going to run the benchmarks using .NET Core 2.1 and .NET 4.8 -``` - -What is going to happen if you provide multiple Full .NET Framework monikers? -Let's say: - -```cmd -dotnet run -c Release -f net461 net472 net48 -``` - -Full .NET Framework always runs every .NET executable using the latest .NET Framework available on a given machine. -If you try to run the benchmarks for a few .NET TFMs, they are all going to be executed using the latest .NET Framework from your machine. -The only difference is that they are all going to have different features enabled depending on the target version they were compiled for. -You can read more about this - [here](https://learn.microsoft.com/dotnet/framework/migration-guide/version-compatibility) and - [here](https://learn.microsoft.com/dotnet/framework/migration-guide/application-compatibility). -This is **.NET Framework behavior which can not be controlled by BenchmarkDotNet or any other tool**. - -**Note:** Console arguments support works only if you pass the `args` to `BenchmarkSwitcher`: - -```cs -class Program -{ - static void Main(string[] args) - => BenchmarkSwitcher - .FromAssembly(typeof(Program).Assembly) - .Run(args); // crucial to make it work -} -``` - -You can achieve the same thing using `[SimpleJobAttribute]`: - -```cs -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Jobs; - -namespace BenchmarkDotNet.Samples -{ - [SimpleJob(RuntimeMoniker.Net48)] - [SimpleJob(RuntimeMoniker.Mono)] - [SimpleJob(RuntimeMoniker.NetCoreApp21)] - [SimpleJob(RuntimeMoniker.NetCoreApp30)] - public class TheClassWithBenchmarks -``` - -Or using a custom config: - -```cs -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Running; - -namespace BenchmarkDotNet.Samples -{ - class Program - { - static void Main(string[] args) - { - var config = DefaultConfig.Instance - .With(Job.Default.With(CoreRuntime.Core21)) - .With(Job.Default.With(CoreRuntime.Core30)) - .With(Job.Default.With(ClrRuntime.Net48)) - .With(Job.Default.With(MonoRuntime.Default)); - - BenchmarkSwitcher - .FromAssembly(typeof(Program).Assembly) - .Run(args, config); - } - } -} -``` - -The recommended way of running the benchmarks for multiple runtimes is to use the `--runtimes` console line argument. -By using the console line argument, you don't need to edit the source code anytime you want to change the list of runtimes. -Moreover, if you share the source code of the benchmark, other people can run it even if they don't have the exact same framework version installed. - -## Official templates for BenchmarkDotNet-based projects - -Since v0.12.0, BenchmarkDotNet provides project templates to setup your benchmarks easily. -The template exists for each major .NET language ([C#](https://learn.microsoft.com/dotnet/csharp/), [F#](https://learn.microsoft.com/dotnet/fsharp/) and [VB](https://learn.microsoft.com/dotnet/visual-basic/)) with equivalent features and structure. -The templates require the [.NET Core SDK](https://www.microsoft.com/net/download). Once installed, run the following command to install the templates: - -```log -dotnet new -i BenchmarkDotNet.Templates -``` - -If you want to uninstall all BenchmarkDotNet templates: - -```log -dotnet new -u BenchmarkDotNet.Templates -``` - -The template is a NuGet package distributed over nuget.org: [BenchmarkDotNet.Templates](https://www.nuget.org/packages/BenchmarkDotNet.Templates/). -To create a new C# benchmark library project from the template, run: - -```log -dotnet new benchmark -``` - -If you'd like to create F# or VB project, you can specify project language with `-lang` option: - -```log -dotnet new benchmark -lang F# -dotnet new benchmark -lang VB -``` - -The template projects have five additional options - all of them are optional. -By default, a class library project targeting netstandard2.0 is created. -You can specify `-f` or `--frameworks` to change target to one or more frameworks: - -```log -dotnet new benchmark -f netstandard2.0;net472 -``` - -The option `--console-app` creates a console app project targeting `netcoreapp3.0` with an entry point: - -```log -dotnet new benchmark --console-app -``` - -This lets you run the benchmarks from a console (`dotnet run`) or from your favorite IDE. -The option `-f` or `--frameworks` will be ignored when `--console-app` is set. -The option `-b` or `--benchmarkName` sets the name of the benchmark class: - -```log -dotnet new benchmark -b Md5VsSha256 -``` - -BenchmarkDotNet lets you create a dedicated configuration class (see [Configs](xref:docs.configs)) to customize the execution of your benchmarks. -To create a benchmark project with a configuration class, use the option `-c` or `--config`: - -```log -dotnet new benchmark -c -``` - -The option `--no-restore` if specified, skips the automatic NuGet restore after the project is created: - -```log -dotnet new benchmark --no-restore -``` - -Use the `-h` or `--help` option to display all possible arguments with a description and the default values: - -```log -dotnet new benchmark --help -``` - -The version of the template NuGet package is synced with the [BenchmarkDotNet](https://www.nuget.org/packages/BenchmarkDotNet/) package. -For instance, the template version `0.12.0` is referencing [BenchmarkDotnet 0.12.0](https://www.nuget.org/packages/BenchmarkDotNet/0.12.0) - there is no floating version behavior. -For more info about the `dotnet new` CLI, please read [the documentation](https://learn.microsoft.com/dotnet/core/tools/dotnet). - -## New NativeMemoryProfiler - -`NativeMemoryProfiler` measure the native memory traffic and adds the extra columns `Allocated native memory` and `Native memory leak` to the summary table. -Internally, it uses `EtwProfiler` to profile the code using ETW. - -Consider the following benchmark: - -```cs -[ShortRunJob] -[NativeMemoryProfiler] -[MemoryDiagnoser] -public class IntroNativeMemory -{ - [Benchmark] - public void BitmapWithLeaks() - { - var flag = new Bitmap(200, 100); - var graphics = Graphics.FromImage(flag); - var blackPen = new Pen(Color.Black, 3); - graphics.DrawLine(blackPen, 100, 100, 500, 100); - } - - [Benchmark] - public void Bitmap() - { - using (var flag = new Bitmap(200, 100)) - { - using (var graphics = Graphics.FromImage(flag)) - { - using (var blackPen = new Pen(Color.Black, 3)) - { - graphics.DrawLine(blackPen, 100, 100, 500, 100); - } - } - } - } - - private const int Size = 20; // Greater value could cause System.OutOfMemoryException for test with memory leaks. - private int ArraySize = Size * Marshal.SizeOf(typeof(int)); - - [Benchmark] - public unsafe void AllocHGlobal() - { - IntPtr unmanagedHandle = Marshal.AllocHGlobal(ArraySize); - Span unmanaged = new Span(unmanagedHandle.ToPointer(), ArraySize); - Marshal.FreeHGlobal(unmanagedHandle); - } - - [Benchmark] - public unsafe void AllocHGlobalWithLeaks() - { - IntPtr unmanagedHandle = Marshal.AllocHGlobal(ArraySize); - Span unmanaged = new Span(unmanagedHandle.ToPointer(), ArraySize); - } -} -``` - -It will produce the summary table like this one: - -| Method | Mean | Error | StdDev | Gen 0 | Gen 1 | Gen 2 | Allocated | Allocated native memory | Native memory leak | -|---------------------- |-------------:|--------------:|-------------:|------:|------:|------:|----------:|------------------------:|-------------------:| -| BitmapWithLeaks | 73,456.43 ns | 22,498.10 ns | 1,233.197 ns | - | - | - | 177 B | 13183 B | 11615 B | -| Bitmap | 91,590.08 ns | 101,468.12 ns | 5,561.810 ns | - | - | - | 180 B | 12624 B | - | -| AllocHGlobal | 79.91 ns | 43.93 ns | 2.408 ns | - | - | - | - | 80 B | - | -| AllocHGlobalWithLeaks | 103.50 ns | 153.21 ns | 8.398 ns | - | - | - | - | 80 B | 80 B | - -As you can see, we have two additional columns `Allocated native memory` and `Native memory leak` that contain some very useful numbers! - -## New ThreadingDiagnoser - -`ThreadingDiagnoser` also adds two extra columns to the summary table: - -* `Completed Work Items`: The number of work items that have been processed in ThreadPool (per single operation) -* `Lock Contentions`: The number of times there **was contention** upon trying to take a Monitor's lock (per single operation) - -Internally, it uses [new APIs](https://github.com/dotnet/corefx/issues/35500) exposed in .NET Core 3.0. - -It can be activated with the help of the `[ThreadingDiagnoser]` attribute: - -```cs -[ThreadingDiagnoser] -public class IntroThreadingDiagnoser -{ - [Benchmark] - public void CompleteOneWorkItem() - { - ManualResetEvent done = new ManualResetEvent(initialState: false); - ThreadPool.QueueUserWorkItem(m => (m as ManualResetEvent).Set(), done); - done.WaitOne(); - } -} -``` - -The above example will print a summary table like this one: - -| Method | Mean | StdDev | Median | Completed Work Items | Lock Contentions | -|-------------------- |--------------:|-----------:|--------------:|---------------------:|-----------------:| -| CompleteOneWorkItem | 8,073.5519 ns | 69.7261 ns | 8,111.6074 ns | 1.0000 | - | - - -## LINQPad 6 support - -Now both LINQPad 5 and LINQPad 6 are supported: - -![](https://user-images.githubusercontent.com/6011991/64907554-5d77b000-d6f4-11e9-9684-50ea678647eb.png) - -## Fast documentation search - -We continue to improve the usability of our documentation. -In this release, we improved the search experience in the documentation: now it works almost instantly with the help of [Algolia](https://www.algolia.com/) engine! -That's how it looks: - -![](https://user-images.githubusercontent.com/2259237/67472874-701dc700-f65a-11e9-85f1-d4f3160f78cf.gif) diff --git a/docs/_changelog/header/v0.12.1.md b/docs/_changelog/header/v0.12.1.md deleted file mode 100644 index 8378e7da70..0000000000 --- a/docs/_changelog/header/v0.12.1.md +++ /dev/null @@ -1,294 +0,0 @@ -## Highlights - -* **.NET 5 support** - As you probably know, .NET Core 5 was officially [rebranded](https://github.com/dotnet/runtime/pull/33694) to .NET 5. - The new version of BenchmarkDotNet supports the new runtime after rebranding. - [#1399](https://github.com/dotnet/BenchmarkDotNet/pull/1399) - [465ebf](https://github.com/dotnet/BenchmarkDotNet/commit/465ebf3fdbf20f0e9219c4c957fb33c13256fdcd) -* **Perfolizer adoption** - The internal statistical engine of BenchmarkDotNet became mature enough to be transformed into an independent project. - Meet [perfolizer](https://github.com/AndreyAkinshin/perfolizer) — a toolkit for performance analysis! - While BenchmarkDotNet focuses on obtaining reliable measurements, perfolizer focuses on the decent analysis of measured data. - You still can use all the statistical algorithms from BenchmarkDotNet, - but you can also install perfolizer as a [standalone NuGet package](https://www.nuget.org/packages/Perfolizer/). - You can find more details in the [official announcement](https://aakinshin.net/posts/introducing-perfolizer/). - [#1386](https://github.com/dotnet/BenchmarkDotNet/pull/1386) - [54a061](https://github.com/dotnet/BenchmarkDotNet/commit/54a06102a6e0cc4169d23c6f9cd2779ee408d2bf) -* **Cross-platform disassembler** - Now the `DisassemblyDiagnoser` is cross-platform! - The disassembling logic was also improved, now it handles runtime helper methods and references to method tables properly. - Internally, it uses the [Iced](https://github.com/0xd4d/iced) library for formatting assembly code. - Special thanks to [@adamsitnik](https://github.com/adamsitnik) for the implementation and [@0xd4d](https://github.com/0xd4d) for Iced! - [#1332](https://github.com/dotnet/BenchmarkDotNet/pull/1332) - [#899](https://github.com/dotnet/BenchmarkDotNet/issues/899) - [#1316](https://github.com/dotnet/BenchmarkDotNet/issues/1316) - [#1364](https://github.com/dotnet/BenchmarkDotNet/issues/1364) - [294320](https://github.com/dotnet/BenchmarkDotNet/commit/294320be9525b0ecfefd0351381756d3a3b77211) -* **EventPipe-based cross-platform profiler** - Now you can easily profiler your benchmarks on Windows, Linux, and macOS! - Just mark your class with the `[EventPipeProfiler(...)]` attribute and get a `.speedscope.json` file that you can browse in [SpeedScope](https://www.speedscope.app/). - Special thanks to [@WojciechNagorski](https://github.com/WojciechNagorski) for the implementation! - [#1321](https://github.com/dotnet/BenchmarkDotNet/pull/1321) - [#1315](https://github.com/dotnet/BenchmarkDotNet/issues/1315) - [c648ff](https://github.com/dotnet/BenchmarkDotNet/commit/c648ff956662abae512c579ffa7f1dc12178f6c3) -* **New fluent API** - We continue to improve our API and make it easier for reading and writing. - Special thanks to [@WojciechNagorski](https://github.com/WojciechNagorski) for the implementation! - [#1273](https://github.com/dotnet/BenchmarkDotNet/pull/1273) - [#1234](https://github.com/dotnet/BenchmarkDotNet/issues/1234) - [640d88](https://github.com/dotnet/BenchmarkDotNet/commit/640d885ae0daddcee7c9ba9b5f1bf5790b9b5ae3) -* **Ref readonly support** - Now you can use `ref readonly` in benchmark signatures. - Special thanks to [@adamsitnik](https://github.com/adamsitnik) for the implementation! - [#1389](https://github.com/dotnet/BenchmarkDotNet/pull/1389) - [#1388](https://github.com/dotnet/BenchmarkDotNet/issues/1388) - [9ac777](https://github.com/dotnet/BenchmarkDotNet/commit/9ac7770682a45afb6cf4ec353f9fa3c69ece67ce) - -## Cross-platform disassembler - -Just mark your benchmark class with the `[DisassemblyDiagnoser]` attribute - and you will get the disassembly listings for all the benchmarks. -The formatting looks pretty nice thanks to [Iced](https://github.com/0xd4d/iced). -It works like a charm on Windows, Linux, and macOS. - -```cs -[DisassemblyDiagnoser] -public class IntroDisassembly -{ - private int[] field = Enumerable.Range(0, 100).ToArray(); - - [Benchmark] - public int SumLocal() - { - var local = field; // we use local variable that points to the field - - int sum = 0; - for (int i = 0; i < local.Length; i++) - sum += local[i]; - - return sum; - } - - [Benchmark] - public int SumField() - { - int sum = 0; - for (int i = 0; i < field.Length; i++) - sum += field[i]; - - return sum; - } -} -``` - -**.NET Core 2.1.16 (CoreCLR 4.6.28516.03, CoreFX 4.6.28516.10), X64 RyuJIT** - -```x86asm -; BenchmarkDotNet.Samples.IntroDisassembly.SumLocal() - mov rax,[rcx+8] - xor edx,edx - xor ecx,ecx - mov r8d,[rax+8] - test r8d,r8d - jle short M00_L01 -M00_L00: - movsxd r9,ecx - add edx,[rax+r9*4+10] - inc ecx - cmp r8d,ecx - jg short M00_L00 -M00_L01: - mov eax,edx - ret -; Total bytes of code 35 -``` - -**.NET Core 2.1.16 (CoreCLR 4.6.28516.03, CoreFX 4.6.28516.10), X64 RyuJIT** - -```x86asm -; BenchmarkDotNet.Samples.IntroDisassembly.SumField() - sub rsp,28 - xor eax,eax - xor edx,edx - mov rcx,[rcx+8] - cmp dword ptr [rcx+8],0 - jle short M00_L01 -M00_L00: - mov r8,rcx - cmp edx,[r8+8] - jae short M00_L02 - movsxd r9,edx - add eax,[r8+r9*4+10] - inc edx - cmp [rcx+8],edx - jg short M00_L00 -M00_L01: - add rsp,28 - ret -M00_L02: - call CORINFO_HELP_RNGCHKFAIL - int 3 -; Total bytes of code 53 -``` - -Now we handle runtime helper methods and references to method tables properly. Example: - -Before: - -```x86asm -; MicroBenchmarks.WithCallsAfter.Benchmark(Int32) - push rsi - sub rsp,20h - mov rsi,rcx - cmp edx,7FFFFFFFh - jne M00_L00 - call MicroBenchmarks.WithCallsAfter.Static() - mov rcx,rsi - call MicroBenchmarks.WithCallsAfter.Instance() - mov rcx,rsi - call MicroBenchmarks.WithCallsAfter.Recursive() - mov rcx,rsi - mov rax,qword ptr [rsi] - mov rax,qword ptr [rax+40h] - call qword ptr [rax+20h] - mov rcx,rsi - mov edx,1 - mov rax,7FF8F4217050h - add rsp,20h - pop rsi - jmp rax -M00_L00: - mov rcx,offset System_Private_CoreLib+0xa31d48 - call coreclr!MetaDataGetDispenser+0x322a0 - mov rsi,rax - mov ecx,0ACFAh - mov rdx,7FF8F42F4680h - call coreclr!MetaDataGetDispenser+0x17140 - mov rdx,rax - mov rcx,rsi - call System.InvalidOperationException..ctor(System.String) - mov rcx,rsi - call coreclr!coreclr_shutdown_2+0x39f0 - int 3 - add byte ptr [rax],al - sbb dword ptr [00007ff9`26284e30],eax - add dword ptr [rax+40h],esp - add byte ptr [rax],al - add byte ptr [rax],al - add byte ptr [rax],al - add byte ptr [rax-70BC4CCh],ah -; Total bytes of code 157 -``` - -After: - -```x86asm -; BenchmarkDotNet.Samples.WithCallsAfter.Benchmark(Int32) - push rsi - sub rsp,20 - mov rsi,rcx - cmp edx,7FFFFFFF - jne M00_L00 - call BenchmarkDotNet.Samples.WithCallsAfter.Static() - mov rcx,rsi - call BenchmarkDotNet.Samples.WithCallsAfter.Instance() - mov rcx,rsi - call BenchmarkDotNet.Samples.WithCallsAfter.Recursive() - mov rcx,rsi - mov rax,[rsi] - mov rax,[rax+40] - call qword ptr [rax+20] - mov rcx,rsi - mov edx,1 - mov rax BenchmarkDotNet.Samples.WithCallsAfter.Benchmark(Boolean) - add rsp,20 - pop rsi - jmp rax -M00_L00: - mov rcx MT_System.InvalidOperationException - call CORINFO_HELP_NEWSFAST - mov rsi,rax - mov ecx,12D - mov rdx,7FF954FF83F0 - call CORINFO_HELP_STRCNS - mov rdx,rax - mov rcx,rsi - call System.InvalidOperationException..ctor(System.String) - mov rcx,rsi - call CORINFO_HELP_THROW - int 3 -; Total bytes of code 134 -``` - -See also: [Cross-runtime .NET disassembly with BenchmarkDotNet](https://aakinshin.net/posts/dotnet-crossruntime-disasm/). - -Special thanks to [@adamsitnik](https://github.com/adamsitnik) for the implementation and [@0xd4d](https://github.com/0xd4d) for Iced! - -## EventPipe-based cross-platform profiler - -Now you can easily profiler your benchmarks on Windows, Linux, and macOS! - -If you want to use the new profiler, you should just mark your benchmark class with the `[EventPipeProfiler(...)]` attribute: - -``` -[EventPipeProfiler(EventPipeProfile.CpuSampling)] // <-- Enables new profiler -public class IntroEventPipeProfiler -{ - [Benchmark] - public void Sleep() => Thread.Sleep(2000); -} -``` - -Once the benchmark run is finished, you get a `.speedscope.json` file that can be opened in [SpeedScope](https://www.speedscope.app/): - -![](https://wojciechnagorski.com/images/EventPipeProfiler/SpeedScope.png#mid) - -The new profiler supports several modes: - -* `CpuSampling` - Useful for tracking CPU usage and general .NET runtime information. This is the default option. -* `GcVerbose` - Tracks GC collections and samples object allocations. -* `GcCollect` - Tracks GC collections only at very low overhead. -* `Jit` - Logging when Just in time (JIT) compilation occurs. Logging of the internal workings of the Just In Time compiler. This is fairly verbose. It details decisions about interesting optimization (like inlining and tail call) - -Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.com/2020/04/cross-platform-profiling-.net-code-with-benchmarkdotnet/) for all the details. - -Special thanks to [@WojciechNagorski](https://github.com/WojciechNagorski) for the implementation! - -## New fluent API - -We continue to improve our API and make it easier for reading and writing. -The old API is still existing, but it is marked as obsolete and will be removed in the further library versions. -The most significant changes: - -**Changes in Job configuration** - -![](https://user-images.githubusercontent.com/17333903/66208963-b49af000-e6b6-11e9-87b3-1e4bb5519273.PNG) - -**Changes in IConfig/ManualConfig** - -![](https://user-images.githubusercontent.com/17333903/66208975-b6fd4a00-e6b6-11e9-94cd-85c73ae0ec27.PNG) - -**Full fluent API** - -![](https://user-images.githubusercontent.com/17333903/66208977-b8c70d80-e6b6-11e9-8e43-6dac3921c121.PNG) - -Special thanks to [@WojciechNagorski](https://github.com/WojciechNagorski) for the implementation! - -## Ref readonly support - -Now you can use `ref readonly` in benchmark signatures. -Here is an example: - -```cs -public class RefReadonlyBenchmark -{ - static readonly int[] array = { 1 }; - - [Benchmark] - public ref readonly int RefReadonly() => ref RefReadonlyMethod(); - - static ref readonly int RefReadonlyMethod() => ref array[0]; -} -``` - -Special thanks to [@adamsitnik](https://github.com/adamsitnik) for the implementation! \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.0.md b/docs/_changelog/header/v0.13.0.md deleted file mode 100644 index d0fca1471d..0000000000 --- a/docs/_changelog/header/v0.13.0.md +++ /dev/null @@ -1,368 +0,0 @@ -It's been a year since our last release. BenchmarkDotNet has been downloaded more than seven million times from [nuget.org](https://www.nuget.org/packages/BenchmarkDotNet/). It's more than we could have ever possibly imagined! Some could say, that it's also more than we can handle ;) That is why we wanted to once again thank [all the contributors](https://github.com/dotnet/BenchmarkDotNet/graphs/contributors) who helped us with `0.13.0` release! - -## Highlights - -In BenchmarkDotNet v0.13.0, we have supported various technologies: - -* .NET 5 and .NET 6 target framework monikers -* .NET SDK installed via snap -* SingleFile deployment -* Xamarin applications -* WASM applications -* Mono AOT - -We have also introduced new features and improvements including: - -* Memory randomization -* Method-specific job attributes -* Sortable parameter columns -* Customizable ratio column -* Improved CoreRun and CoreRT support -* Improved Hardware Counters support - -Of course, this release includes dozens of other improvements and bug fixes! - -## Supported technologies - -### .NET 5, .NET 6, SingleFile and snap - -At some point in time, `netcoreapp5.0` moniker was changed to `net5.0`, which required a fix on our side ([#1479](https://github.com/dotnet/BenchmarkDotNet/pull/1479), btw we love this kind of changes). Moreover, .NET 5 introduced platform-specific TMFs (example: `net5.0-windows10.0.19041.0`) which also required some extra work: [#1560](https://github.com/dotnet/BenchmarkDotNet/pull/1560), [#1691](https://github.com/dotnet/BenchmarkDotNet/pull/1691). - - - In [#1523](https://github.com/dotnet/BenchmarkDotNet/pull/1523) support for .NET 6 was added. - -```xml -net5.0;net5.0-windows10.0.19041.0;net6.0 -``` - -In [#1686](https://github.com/dotnet/BenchmarkDotNet/pull/1686) [@am11](https://github.com/am11) has implemented support for **single file deployment** (supported in .NET 5 onwards). - -Last, but not least in [#1652](https://github.com/dotnet/BenchmarkDotNet/pull/1652) **snap** support has been implemented. - -```log -adam@adsitnik-ubuntu:~/projects/BenchmarkDotNet/samples/BenchmarkDotNet.Samples$ dotnet50 run -c Release -f net5.0 --filter BenchmarkDotNet.Samples.IntroColdStart.Foo -// Validating benchmarks: -// ***** BenchmarkRunner: Start ***** -// ***** Found 1 benchmark(s) in total ***** -// ***** Building 1 exe(s) in Parallel: Start ***** -// start /snap/dotnet-sdk/112/dotnet restore /p:UseSharedCompilation=false /p:BuildInParallel=false /m:1 /p:Deterministic=true /p:Optimize=true in /home/adam/projects/BenchmarkDotNet/samples/BenchmarkDotNet.Samples/bin/Release/net5.0/9a018ee4-0f33-46dd-9093-01d3bf31233b -// command took 1.49s and exited with 0 -// start /snap/dotnet-sdk/112/dotnet build -c Release --no-restore /p:UseSharedCompilation=false /p:BuildInParallel=false /m:1 /p:Deterministic=true /p:Optimize=true in /home/adam/projects/BenchmarkDotNet/samples/BenchmarkDotNet.Samples/bin/Release/net5.0/9a018ee4-0f33-46dd-9093-01d3bf31233b -// command took 2.78s and exited with 0 -// ***** Done, took 00:00:04 (4.37 sec) ***** -// Found 1 benchmarks: -// IntroColdStart.Foo: Job-NECTOD(IterationCount=5, RunStrategy=ColdStart) - -// ************************** -// Benchmark: IntroColdStart.Foo: Job-NECTOD(IterationCount=5, RunStrategy=ColdStart) -// *** Execute *** -// Launch: 1 / 1 -// Execute: /snap/dotnet-sdk/112/dotnet "9a018ee4-0f33-46dd-9093-01d3bf31233b.dll" --benchmarkName "BenchmarkDotNet.Samples.IntroColdStart.Foo" --job "IterationCount=5, RunStrategy=ColdStart" --benchmarkId 0 in /home/adam/projects/BenchmarkDotNet/samples/BenchmarkDotNet.Samples/bin/Release/net5.0/9a018ee4-0f33-46dd-9093-01d3bf31233b/bin/Release/net5.0 -``` - -### Xamarin support - -Thanks to the contributions of the amazing [@jonathanpeppers](https://github.com/jonathanpeppers) BenchmarkDotNet supports Xamarin! The examples can be found in our repo: [iOS](https://github.com/dotnet/BenchmarkDotNet/tree/master/samples/BenchmarkDotNet.Samples.iOS), [Android](https://github.com/dotnet/BenchmarkDotNet/tree/master/samples/BenchmarkDotNet.Samples.Android). - -![](https://user-images.githubusercontent.com/840039/73184210-b105fb80-40e1-11ea-9efb-d29fe4d08c79.png#mid) - -[#1360](https://github.com/dotnet/BenchmarkDotNet/pull/1360), [#1429](https://github.com/dotnet/BenchmarkDotNet/pull/1429), [#1434](https://github.com/dotnet/BenchmarkDotNet/pull/1434), [#1509](https://github.com/dotnet/BenchmarkDotNet/pull/1509) - -### WASM support - -Thanks to the work of [@naricc](https://github.com/naricc) you can now benchmark WASM using Mono Runtime! For more details, please refer to our [docs](https://benchmarkdotnet.org/articles/configs/toolchains.html#Wasm). - -![](https://user-images.githubusercontent.com/6011991/87671577-dd860080-c771-11ea-922f-e26a32a3f831.png#mid) - -[#1483](https://github.com/dotnet/BenchmarkDotNet/pull/1483), [#1498](https://github.com/dotnet/BenchmarkDotNet/pull/1498), [#1500](https://github.com/dotnet/BenchmarkDotNet/pull/1500), [#1501](https://github.com/dotnet/BenchmarkDotNet/pull/1501), [#1507](https://github.com/dotnet/BenchmarkDotNet/pull/1507), [#1592](https://github.com/dotnet/BenchmarkDotNet/pull/1592), [#1689](https://github.com/dotnet/BenchmarkDotNet/pull/1689). - -### Mono AOT support - -In another awesome contribution ([#1662](https://github.com/dotnet/BenchmarkDotNet/pull/1662)) [@naricc](https://github.com/naricc) has implemented Mono AOT support. The new toolchain supports doing Mono AOT runs with both the Mono-Mini compiler and the Mono-LLVM compiler (which uses LLVM on the back end). - - For more details, please go to our [docs](https://benchmarkdotnet.org/articles/configs/toolchains.html#MonoAotLLVM). - -## New features and improvements - -### Memory randomization - -In [#1587](https://github.com/dotnet/BenchmarkDotNet/pull/1587) [@adamsitnik](https://github.com/adamsitnik) has introduced a new, **experimental** feature called "Memory Randomization". - -This feature allows you to ask BenchmarkDotNet to randomize the memory alignment by allocating random-sized byte arrays between iterations and **call [GlobalSetup] before every benchmark iteration** and `[GlobalCleanup]` after every benchmark iteration. - -Sample benchmark: - -```cs -public class IntroMemoryRandomization -{ - [Params(512 * 4)] - public int Size; - - private int[] _array; - private int[] _destination; - - [GlobalSetup] - public void Setup() - { - _array = new int[Size]; - _destination = new int[Size]; - } - - [Benchmark] - public void Array() => System.Array.Copy(_array, _destination, Size); -} -``` - -Without asking for the randomization, the objects are allocated in `[GlobalSetup]` and their unmodified addresses (and alignment) are used for all iterations (as long as they are not promoted to an older generation by the GC). This is typically the desired behavior, as it gives you very nice and flat distributions: - -```cmd -dotnet run -c Release --filter IntroMemoryRandomization -``` - -```ini --------------------- Histogram -------------------- -[502.859 ns ; 508.045 ns) | @@@@@@@@@@@@@@@ ---------------------------------------------------- -``` - -But if for some reason you are interested in getting a distribution that is better reflecting the "real-life" performance you can enable the randomization: - -```cmd -dotnet run -c Release --filter IntroMemoryRandomization --memoryRandomization true -``` - -```ini --------------------- Histogram -------------------- -[108.803 ns ; 213.537 ns) | @@@@@@@@@@@@@@@ -[213.537 ns ; 315.458 ns) | -[315.458 ns ; 446.853 ns) | @@@@@@@@@@@@@@@@@@@@ -[446.853 ns ; 559.259 ns) | @@@@@@@@@@@@@@@ ---------------------------------------------------- -``` - -### Method-specific job attributes - -From now, all attributes that derive from `JobMutatorConfigBaseAttribute` ([full list](https://benchmarkdotnet.org/api/BenchmarkDotNet.Attributes.JobMutatorConfigBaseAttribute.html#BenchmarkDotNet_Attributes_JobMutatorConfigBaseAttribute)) can be applied to methods. You no longer have to move a method to a separate type to customize config for it. - -```cs -[Benchmark] -[WarmupCount(1)] -public void SingleWarmupIteration() - -[Benchmark] -[WarmupCount(9)] -public void NineWarmupIterations() -``` - -### Sortable parameter columns - -In order to sort columns of parameters in the results table, you can use the Property `Priority` inside the params attribute. The priority range is `[Int32.MinValue;Int32.MaxValue]`, lower priorities will appear earlier in the column order. The default priority is set to `0`. - -```cs -public class IntroParamsPriority -{ - [Params(100)] - public int A { get; set; } - - [Params(10, Priority = -100)] - public int B { get; set; } - - [Benchmark] - public void Benchmark() => Thread.Sleep(A + B + 5); -} -``` - -| Method | B | A | Mean | Error | StdDev | -|---------- |--- |---- |---------:|--------:|--------:| -| Benchmark | 10 | 100 | 115.4 ms | 0.12 ms | 0.11 ms | - - -This feature got implemented by [@JohannesDeml](https://github.com/JohannesDeml) in [#1612](https://github.com/dotnet/BenchmarkDotNet/pull/1612). - -### Customizable ratio column - -Now it's possible to customize the format of the ratio column. - -```cs -[Config(typeof(Config))] -public class IntroRatioStyle -{ - [Benchmark(Baseline = true)] - public void Baseline() => Thread.Sleep(1000); - - [Benchmark] - public void Bar() => Thread.Sleep(150); - - [Benchmark] - public void Foo() => Thread.Sleep(1150); - - private class Config : ManualConfig - { - public Config() - { - SummaryStyle = SummaryStyle.Default.WithRatioStyle(RatioStyle.Trend); - } - } -} -``` - -``` -| Method | Mean | Error | StdDev | Ratio | RatioSD | -|--------- |-----------:|--------:|--------:|-------------:|--------:| -| Baseline | 1,000.6 ms | 2.48 ms | 0.14 ms | baseline | | -| Bar | 150.9 ms | 1.30 ms | 0.07 ms | 6.63x faster | 0.00x | -| Foo | 1,150.4 ms | 5.17 ms | 0.28 ms | 1.15x slower | 0.00x | -``` - -This feature was implemented in [#731](https://github.com/dotnet/BenchmarkDotNet/issues/721). - -### Improved CoreRun support - -BenchmarkDotNet was reporting invalid .NET Core version number when comparing performance using CoreRuns built from `dotnet/corefx` and `dotnet/runtime`. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1580](https://github.com/dotnet/BenchmarkDotNet/pull/1580) - -In [#1552](https://github.com/dotnet/BenchmarkDotNet/pull/1552) [@stanciuadrian](https://github.com/stanciuadrian) has implemented support for all `GcMode` characteristics for `CoreRunToolchain`. Previously the settings were just ignored, now they are being translated to corresponding `COMPlus_*` env vars. - -### Improved CoreRT support - -CoreRT has moved from https://github.com/dotnet/corert/ to https://github.com/dotnet/runtimelab/tree/feature/NativeAOT and we had to update the default compiler version and nuget feed address. Moreover, there was a bug in `CoreRtToolchain` which was causing any additional native dependencies to not work. - -Big thanks to [@MichalStrehovsky](https://github.com/MichalStrehovsky), [@jkotas](https://github.com/jkotas) and [@kant2002](https://github.com/kant2002) for their help and support! - -[#1606](https://github.com/dotnet/BenchmarkDotNet/pull/1606), [#1643](https://github.com/dotnet/BenchmarkDotNet/pull/1643), [#1679](https://github.com/dotnet/BenchmarkDotNet/pull/1679) - -### Command-line argument support in BenchmarkRunner - -So far only `BenchmarkSwitcher` was capable of handling console line arguments. Thanks to [@chan18](https://github.com/chan18) `BenchmarkRunner` supports them as well ([#1292](https://github.com/dotnet/BenchmarkDotNet/pull/1292)): - -```cs -public class Program -{ - public static void Main(string[] args) => BenchmarkRunner.Run(typeof(Program).Assembly, args: args); -} -``` - -### New API: ManualConfig.CreateMinimumViable - -`ManualConfig.CreateEmpty` creates a completely empty config. Without adding a column provider and a logger to the config the users won't see any results being printed. In [#1582](https://github.com/dotnet/BenchmarkDotNet/pull/1582) [@adamsitnik](https://github.com/adamsitnik) has introduced a new method that creates minimum viable config: - -```cs -IConfig before = ManualConfig.CreateEmpty() - .AddColumnProvider(DefaultColumnProviders.Instance) - .AddLogger(ConsoleLogger.Default); - -IConfig after = ManualConfig.CreateMinimumViable(); -``` - -### Benchmarking NuGet packages from custom feeds - -In [#1659](https://github.com/dotnet/BenchmarkDotNet/pull/1659/) [@workgroupengineering](https://github.com/workgroupengineering) added the possibility to indicate the source of the tested nuget package and whether it is a pre-release version. - -### Deterministic benchmark builds - -BenchmarkDotNet is now always enforcing Deterministic builds ([#1489](https://github.com/dotnet/BenchmarkDotNet/pull/1489)) and Optimizations enabled ([#1494](https://github.com/dotnet/BenchmarkDotNet/pull/1494)) which is a must-have if you are using custom build configurations. MSBuild enforces optimizations **only** for configurations that are named `Release` (the comparison is case-insensitive). - -```xml - - - - - - -``` - -```cs -var config = DefaultConfig.Instance - .AddJob(Job.Default.WithCustomBuildConfiguration("X").WithId("X").AsBaseline()) - .AddJob(Job.Default.WithCustomBuildConfiguration("Y").WithId("Y")); -``` - -[#1489](https://github.com/dotnet/BenchmarkDotNet/pull/1489), [#1494](https://github.com/dotnet/BenchmarkDotNet/pull/1494) - -### Improved Hardware Counters support - -BenchmarkDotNet is being used by the .NET Team to ensure that .NET is not regressing. More than three thousand benchmarks (they can be found [here](https://github.com/dotnet/performance/tree/main/src/benchmarks/micro)) are being executed multiple times a day on multiple hardware configs. Recently, .NET Team started to use `InstructionsRetired` to help to filter unstable benchmarks that report regressions despite not changing the number of instructions retired. This has exposed few bugs in Hardware Counters support in BenchmarkDotNet, which all got fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1547](https://github.com/dotnet/BenchmarkDotNet/pull/1547) and [#1550](https://github.com/dotnet/BenchmarkDotNet/pull/1550). Moreover, we have **removed** the old [PmcDiagnoser](https://adamsitnik.com/Hardware-Counters-Diagnoser/) and extended [EtwProfiler](https://adamsitnik.com/ETW-Profiler/) with the hardware counters support. It's just much more accurate and futureproof. For details, please go to [#1548](https://github.com/dotnet/BenchmarkDotNet/pull/1548). - -How stable was `PmcDiagnoser` (same benchmarks run twice in a row on the same machine): - -| Method | Runtime | InstructionRetired/Op | -|---------- |--------------------- |----------------------:| -| Burgers_0 | .NET 5.0 | 845,746 | -| Burgers_0 | .NET Core 2.1 | 30,154,151 | -| Burgers_0 | .NET Framework 4.6.1 | 4,230,848 | - -| Method | Runtime | InstructionRetired/Op | -|---------- |--------------------- |----------------------:| -| Burgers_0 | .NET 5.0 | 34,154,524 | -| Burgers_0 | .NET Core 2.1 | 246,534,203 | -| Burgers_0 | .NET Framework 4.6.1 | 2,607,686 | - -How stable is `EtwProfiler`: - -| Method | Runtime | InstructionRetired/Op | -|---------- |--------------------- |----------------------:| -| Burgers_0 | .NET 5.0 | 3,069,978,261 | -| Burgers_0 | .NET Core 2.1 | 3,676,000,000 | -| Burgers_0 | .NET Framework 4.6.1 | 3,468,866,667 | - -| Method | Runtime | InstructionRetired/Op | -|---------- |--------------------- |----------------------:| -| Burgers_0 | .NET 5.0 | 3,066,810,000 | -| Burgers_0 | .NET Core 2.1 | 3,674,666,667 | -| Burgers_0 | .NET Framework 4.6.1 | 3,468,600,000 | - -Moreover, in [#1540](https://github.com/dotnet/BenchmarkDotNet/pull/1540) [@WojciechNagorski](https://github.com/WojciechNagorski) has added the removal of temporary files created by `EtwProfiler`. - -### Improved Troubleshooting - -We have the possibility to ask BDN to stop on the first error: `--stopOnFirstError true`. - -The problem was when the users had not asked for that, tried to run `n` benchmarks, all of them failed to build, and BDN was printing the same build error `n` times. - -In [#1672](https://github.com/dotnet/BenchmarkDotNet/pull/1672) [@adamsitnik](https://github.com/adamsitnik) has changed that, so when all the build fails, BDN stops after printing the first error. - -Moreover, we have also changed the default behavior for the failed builds of the boilerplate code. If the build fails, we don't remove the files. Previously we have required the users to pass `--keepFiles` to keep them. See [#1567](https://github.com/dotnet/BenchmarkDotNet/pull/1567) for more details and don't forget about the [Troubleshooting](https://benchmarkdotnet.org/articles/guides/troubleshooting.html) docs! - -### Docs and Samples improvements - -Big thanks to [@lukasz-pyrzyk](https://github.com/lukasz-pyrzyk), [@fleckert](https://github.com/fleckert), [@MarecekF](https://github.com/MarecekF), [@joostas](https://github.com/joostas), [@michalgalecki](https://github.com/michalgalecki), [@WojciechNagorski](https://github.com/WojciechNagorski), [@MendelMonteiro](https://github.com/MendelMonteiro), [@kevinsalimi](https://github.com/kevinsalimi), [@cedric-cf](https://github.com/cedric-cf), [@YohDeadfall](https://github.com/YohDeadfall), [@jeffhandley](https://github.com/jeffhandley) and [@JohannesDeml](https://github.com/JohannesDeml) who have improved our docs and samples! - -[#1463](https://github.com/dotnet/BenchmarkDotNet/pull/1463), [#1465](https://github.com/dotnet/BenchmarkDotNet/pull/1465), [#1508](https://github.com/dotnet/BenchmarkDotNet/pull/1508), [#1518](https://github.com/dotnet/BenchmarkDotNet/pull/1518), [#1554](https://github.com/dotnet/BenchmarkDotNet/pull/1554), [#1568](https://github.com/dotnet/BenchmarkDotNet/pull/1568), [#1601](https://github.com/dotnet/BenchmarkDotNet/pull/1601), [#1633](https://github.com/dotnet/BenchmarkDotNet/pull/1633), [#1645](https://github.com/dotnet/BenchmarkDotNet/pull/1645), [#1647](https://github.com/dotnet/BenchmarkDotNet/pull/1647), [#1657](https://github.com/dotnet/BenchmarkDotNet/pull/1657), [#1675](https://github.com/dotnet/BenchmarkDotNet/pull/1675), [#1676](https://github.com/dotnet/BenchmarkDotNet/pull/1676), [#1690](https://github.com/dotnet/BenchmarkDotNet/pull/1690). - -### Template improvements - -* Projects created out of our official templates might have been unexpectedly packed when running `dotnet pack` on the entire solution. In [#1584](https://github.com/dotnet/BenchmarkDotNet/pull/1584) [@kendaleiv](https://github.com/kendaleiv) has explicitly disabled packing for the template. -* The template had `netcoreapp3.0` TFM hardcoded. This got fixed by [@https://github.com/ExceptionCaught](https://github.com/ExceptionCaught) in [#1630](https://github.com/dotnet/BenchmarkDotNet/pull/1630) and [#1632](https://github.com/dotnet/BenchmarkDotNet/pull/1632). -* In [#1667](https://github.com/dotnet/BenchmarkDotNet/pull/1667) [@YohDeadfall](https://github.com/YohDeadfall) has changed the default debug type from `portable` to `pdbonly` (required by `DisassemblyDiagnoser`). - -## Bug fixes - - * Very long string `[Arguments]` and `[Params]` were causing BenchmarkDotNet to crash. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1248](https://github.com/dotnet/BenchmarkDotNet/pull/1248) and [#1545](https://github.com/dotnet/BenchmarkDotNet/pull/1545/). So far trace file names were containing full benchmark names and arguments. Now if the name is too long, the trace file name is a hash (consistent for multiple runs of the same benchmark). The same goes for passing benchmark name by the host process to the benchmark process via command-line arguments. -* `LangVersion` set to a non-numeric value like `latest` was crashing the build. Fixed by [@martincostello](https://github.com/martincostello) in [#1420](https://github.com/dotnet/BenchmarkDotNet/pull/1420). -* Windows 10 November 201**9** was being recognized as 201**8**. Fixed by [@kapsiR](https://github.com/kapsiR) in [#1437](https://github.com/dotnet/BenchmarkDotNet/pull/1437). -* Assemblies loaded via streams were not supported. Fixed by [@jeremyosterhoudt](https://github.com/jeremyosterhoudt) in [#1443](https://github.com/dotnet/BenchmarkDotNet/pull/1443). -* [NativeMemoryProfiler](https://wojciechnagorski.com/2019/08/analyzing-native-memory-allocation-with-benchmarkdotnet/) was detecting small leaks that were false positives. Fixed by [@WojciechNagorski](https://github.com/WojciechNagorski) in [#1451](https://github.com/dotnet/BenchmarkDotNet/pull/1451) and [#1600](https://github.com/dotnet/BenchmarkDotNet/pull/1600). -* [DisassemblyDiagnoser](https://adamsitnik.com/Disassembly-Diagnoser/) was crashing on Linux. Fixed by [@damageboy](https://github.com/damageboy) in [#1459](https://github.com/dotnet/BenchmarkDotNet/pull/1459). -* Target framework moniker was being printed as toolchain name for Full .NET Framework benchmarks. Fixed by [@svick](https://github.com/svick) in [#1471](https://github.com/dotnet/BenchmarkDotNet/pull/1471). -* `[ParamsSource]` returning `IEnumerable` was not working properly when combined with `[Arguments]`. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1478](https://github.com/dotnet/BenchmarkDotNet/pull/1478). -* `NullReferenceException` in `MultimodalDistributionAnalyzer`. Fixed by [@suslovk](https://github.com/suslovk) in [#1488](https://github.com/dotnet/BenchmarkDotNet/pull/1488). -* `NotSupportedException` was being thrown if there was an encoding mismatch between host and benchmark process. Diagnosed by [@ChristophLindemann](https://github.com/ChristophLindemann) in [#1487](https://github.com/dotnet/BenchmarkDotNet/issues/1487), fixed by [@lovettchris](https://github.com/lovettchris) in [#1491](https://github.com/dotnet/BenchmarkDotNet/pull/1491). -* `MissingMethodException` was being thrown in projects that referenced a newer version of [Iced](https://github.com/icedland/iced). Fixed by [@Symbai](https://github.com/Symbai) in [#1497](https://github.com/dotnet/BenchmarkDotNet/pull/1497) and in [#1502](https://github.com/dotnet/BenchmarkDotNet/pull/1502). -* `AppendTargetFrameworkToOutputPath` set to `false` was not supported. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1563](https://github.com/dotnet/BenchmarkDotNet/pull/1563) -* A locking finalizer could have hanged benchmark process which would just print `// AfterAll` and never quit. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1571](https://github.com/dotnet/BenchmarkDotNet/pull/1571). To prevent other hangs from happening, a timeout of `250ms` was added. If the process does not quit after running the benchmarks and waiting `250ms`, it's being force killed. -* In some cases, `JsonExporter` was reporting `NaN` for some of the Statistics. This was breaking non-.NET JSON deserializers. Fixed by [@marcnet80](https://github.com/marcnet80) in [#1581](https://github.com/dotnet/BenchmarkDotNet/pull/1581). -* `UnitType.Size` metrics were not using the provided number format. Fixed by [@jodydonetti](https://github.com/jodydonetti) in [#1569](https://github.com/dotnet/BenchmarkDotNet/pull/1569) and [#1618](https://github.com/dotnet/BenchmarkDotNet/pull/1618). -* `MaxColumnWidth` setting was not used for type names. Fixed by [@JohannesDeml](https://github.com/JohannesDeml) in [#1609](https://github.com/dotnet/BenchmarkDotNet/pull/1609). -* Current culture was not respected when formatting `Ratio` column values. Fixed by [@JohannesDeml](https://github.com/JohannesDeml) in [#1610](https://github.com/dotnet/BenchmarkDotNet/pull/1610). -* BenchmarkDotNet was redirecting Standard Error of the benchmark process, which was causing deadlocks for benchmarks that were writing to it. Fixed by [@adamstinik](https://github.com/adamsitnik) in [#1631](https://github.com/dotnet/BenchmarkDotNet/pull/1631) -* `DisassemblyDiagnoser` was failing to disassemble multiline source code. [@YohDeadfall](https://github.com/YohDeadfall) fixed that in [#1674](https://github.com/dotnet/BenchmarkDotNet/pull/1674). -* In [#1644](https://github.com/dotnet/BenchmarkDotNet/pull/1644) [@adamstinik](https://github.com/adamsitnik) has fixed the inconsistency between benchmark filter and hint. - -## Removal of the dotnet global tool - -In [#1006](https://github.com/dotnet/BenchmarkDotNet/pull/1006) (0.11.4) we have introduced a new dotnet global tool. - -By looking at the number of reported bugs we got to the conclusion that the tool has not passed the test of time. - -Why? Because it was based entirely on dynamic assembly loading which is very hard to get right in .NET and the fact that we support all existing .NET Runtimes (.NET, .NET Core, Mono, CoreRT) made it even harder (if not impossible). - -**We have removed it and the old versions are no longer supported**. For more details, please refer to [#1572](https://github.com/dotnet/BenchmarkDotNet/pull/1572). \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.1.md b/docs/_changelog/header/v0.13.1.md deleted file mode 100644 index 5744a4f2e2..0000000000 --- a/docs/_changelog/header/v0.13.1.md +++ /dev/null @@ -1,15 +0,0 @@ -BenchmarkDotNet v0.13.1 is a service update with various bug fixes and improvements. - -## Highlights - -* S390x architecture support ([#1712](https://github.com/dotnet/BenchmarkDotNet/pull/1712)) -* Various WASM toolchain improvements ([#1719](https://github.com/dotnet/BenchmarkDotNet/pull/1719), [#1722](https://github.com/dotnet/BenchmarkDotNet/pull/1722), [#1729](https://github.com/dotnet/BenchmarkDotNet/pull/1729), [#1742](https://github.com/dotnet/BenchmarkDotNet/pull/1742), [#1743](https://github.com/dotnet/BenchmarkDotNet/pull/1743), [#1744](https://github.com/dotnet/BenchmarkDotNet/pull/1744), [#1746](https://github.com/dotnet/BenchmarkDotNet/pull/1746), [#1757](https://github.com/dotnet/BenchmarkDotNet/pull/1757), [#1763](https://github.com/dotnet/BenchmarkDotNet/pull/1763)) -* Support of CoreRT on 5.0.3XX SDK ([#1725](https://github.com/dotnet/BenchmarkDotNet/pull/1725)) -* Using Utf8 for reading from standard input (fixes a nasty encoding-related bug) ([#1735](https://github.com/dotnet/BenchmarkDotNet/pull/1735)) -* Adjusting WaitForExit timeouts ([#1745](https://github.com/dotnet/BenchmarkDotNet/pull/1745)) -* Support for returning unmanaged types from benchmarks with InProcessToolchain ([#1750](https://github.com/dotnet/BenchmarkDotNet/pull/1750)) -* Disabled Tiered JIT ([#1751](https://github.com/dotnet/BenchmarkDotNet/pull/1751)) -* A MemoryDiagnoser bug fix ([#1762](https://github.com/dotnet/BenchmarkDotNet/pull/1762)) -* Allow users to hide Gen X columns ([#1764](https://github.com/dotnet/BenchmarkDotNet/pull/1764)) -* Copy GC settings from host process to the benchmark process ([#1765](https://github.com/dotnet/BenchmarkDotNet/pull/1765)) -* Do not split surrogates in shortified parameter values ([#1705](https://github.com/dotnet/BenchmarkDotNet/pull/1705)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.10.md b/docs/_changelog/header/v0.13.10.md deleted file mode 100644 index 9f55725d19..0000000000 --- a/docs/_changelog/header/v0.13.10.md +++ /dev/null @@ -1,3 +0,0 @@ -## Highlights - -Initial support of .NET 9 and minor bug fixes. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.2.md b/docs/_changelog/header/v0.13.2.md deleted file mode 100644 index dd5365e56e..0000000000 --- a/docs/_changelog/header/v0.13.2.md +++ /dev/null @@ -1,254 +0,0 @@ -## Highlights - -In BenchmarkDotNet v0.13.2, we have implemented support for: - -* .NET 7 -* NativeAOT -* .NET Framework 4.8.1 - -We have also introduced new features and improvements including: - -* Possibility to hide selected columns, -* Allocation Ratio column, -* Logging progress and estimated finish time, -* ARM64 support for `BenchmarkDotNet.Diagnostics.Windows` package, -* Printing Hardware Intrinsics information, -* Glob filters support for DisassemblyDiagnoser. - -Of course, this release includes dozens of other improvements and bug fixes! - -Our special thanks go to [@mawosoft](https://github.com/mawosoft), [@YegorStepanov](https://github.com/YegorStepanov) and [@radical](https://github.com/radical) who fixed a LOT of really nasty bugs. - -## Supported technologies - -### .NET 7 and .NET Framework 4.8.1 - -.NET 4.8.1 has been [released](https://devblogs.microsoft.com/dotnet/announcing-dotnet-framework-481/) earlier this month, while .NET 7 should land in autumn this year. Now you can use BenchmarkDotNet to compare both! - -```ini -BenchmarkDotNet=v0.13.1.1845-nightly, OS=Windows 11 (10.0.22622.575) -Microsoft SQ1 3.0 GHz, 1 CPU, 8 logical and 8 physical cores -.NET SDK=7.0.100-preview.6.22352.1 - [Host] : .NET 7.0.0 (7.0.22.32404), Arm64 RyuJIT AdvSIMD - Job-QJVIDT : .NET 7.0.0 (7.0.22.32404), Arm64 RyuJIT AdvSIMD - Job-FNNXOY : .NET Framework 4.8.1 (4.8.9032.0), Arm64 RyuJIT -``` - -| Method | Runtime | Mean | Allocated | -|-------------- |--------------------- |---------:|----------:| -| BinaryTrees_2 | .NET 7.0 | 193.6 ms | 227.33 MB | -| BinaryTrees_2 | .NET Framework 4.8.1 | 192.8 ms | 228.01 MB | - -Credit for adding .NET 7 support in [#1816](https://github.com/dotnet/BenchmarkDotNet/pull/1816) goes to [@am11](https://github.com/am11). [@adamsitnik](https://github.com/adamsitnik) implemented .NET 4.8.1 support in [#2044](https://github.com/dotnet/BenchmarkDotNet/pull/2044) and [#2067](https://github.com/dotnet/BenchmarkDotNet/pull/2067). Big thanks to [@MichalPetryka](https://github.com/MichalPetryka) who was using preview versions of BenchmarkDotNet and reported a bug related to .NET 4.8.1 support: [#2059](https://github.com/dotnet/BenchmarkDotNet/issues/2059) that got fixed before we released a new version. - -### NativeAOT - -We are really excited to see the experimental CoreRT project grow and become officially supported by Microsoft (under new name: NativeAOT)! You can read more about it [here](https://devblogs.microsoft.com/dotnet/announcing-dotnet-7-preview-3/#what-is-native-aot). Implementing and improving the support was a combined effort of multiple contributors that spawned across multiple repositories: -* [@MichalStrehovsky](https://github.com/MichalStrehovsky): [#66290 in dotnet/runtime](https://github.com/dotnet/runtime/pull/66290), [#2020 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/2020), [#2046 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/2046) -* [@hez2010](https://github.com/hez2010): [#66650 in dotnet/runtime](https://github.com/dotnet/runtime/pull/66650) -* [@Beau-Gosse-dev](https://github.com/Beau-Gosse-dev): [#1955 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1955) -* [@adamsitnik](https://github.com/adamsitnik): [#1960 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1960), [#1965 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1965), [#1972 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1972), [#1973 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1973), [#1994 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1994), [#1997 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1997), [#2045 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/2045), [#2068 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/2068) -* [@kant2002](https://github.com/kant2002): [#1976 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1976), [#1979 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1979) -* [@jkotas](https://github.com/jkotas): [#68038 in dotnet/runtime](https://github.com/dotnet/runtime/pull/68038), [#68142 in dotnet/runtime](https://github.com/dotnet/runtime/pull/68142), [#68249 in dotnet/runtime](https://github.com/dotnet/runtime/pull/68249), [#68308 in dotnet/runtime](https://github.com/dotnet/runtime/pull/68308), [#68375 in dotnet/runtime](https://github.com/dotnet/runtime/pull/68375) -* [@MichalPetryka](https://github.com/MichalPetryka): [#2065 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/2065) - - -As every AOT solution, NativeAOT has some [limitations](https://github.com/dotnet/runtime/blob/main/src/coreclr/nativeaot/docs/limitations.md) like limited reflection support or lack of dynamic assembly loading. Because of that, the host process (what you run from command line) is never an AOT process, but just a regular .NET process. This process (called Host process) uses reflection to read benchmarks metadata (find all `[Benchmark]` methods etc.), generates a new project that references the benchmarks and compiles it using ILCompiler. The boilerplate code is not using reflection, so the project is built with `TrimmerDefaultAction=link` (we have greatly reduced build time thanks to that). Such compilation produces a native executable, which is later started by the Host process. This process (called Benchmark or Child process) performs the actual benchmarking and reports the results back to the Host process. By default BenchmarkDotNet uses the latest version of `Microsoft.DotNet.ILCompiler` to build the NativeAOT benchmark according to [this instructions](https://github.com/dotnet/runtime/blob/main/src/coreclr/nativeaot/docs/compiling.md). Moreover, BenchmarkDotNet by default uses current machines CPU features (change: [#1994](https://github.com/dotnet/BenchmarkDotNet/pull/1994), discussion: [#2061](https://github.com/dotnet/BenchmarkDotNet/issues/2061)) and if you don't like this behavior, you can [disable it](https://github.com/dotnet/BenchmarkDotNet/issues/2061#issuecomment-1203602177). - -This is why you need to: -- install [pre-requisites](https://learn.microsoft.com/dotnet/core/deploying/native-aot/#prerequisites) required by NativeAOT compiler -- target .NET to be able to run NativeAOT benchmarks (example: `net7.0` in the .csproj file) -- run the app as a .NET process (example: `dotnet run -c Release -f net7.0`). -- specify the NativeAOT runtime in an explicit way, either by using console line arguments `--runtimes nativeaot7.0` (the recommended approach), or by using`[SimpleJob]` attribute or by using the fluent Job config API `Job.ShortRun.With(NativeAotRuntime.Net70)`: - -```cmd -dotnet run -c Release -f net7.0 --runtimes nativeaot7.0 -``` - -For more examples please go to [docs](https://benchmarkdotnet.org/articles/configs/toolchains.html#nativeaot). - -```ini -BenchmarkDotNet=v0.13.1.1845-nightly, OS=Windows 11 (10.0.22000.856/21H2) -AMD Ryzen Threadripper PRO 3945WX 12-Cores, 1 CPU, 24 logical and 12 physical cores -.NET SDK=7.0.100-rc.1.22423.16 - [Host] : .NET 7.0.0 (7.0.22.42223), X64 RyuJIT AVX2 - Job-KDVXET : .NET 7.0.0 (7.0.22.42223), X64 RyuJIT AVX2 - Job-HFRAGK : .NET 7.0.0-rc.1.22424.9, X64 NativeAOT AVX2 -``` - -| Method | Runtime | Mean | Ratio | Allocated | -|-------------- |-------------- |---------:|------:|----------:| -| BinaryTrees_2 | .NET 7.0 | 95.06 ms | 1.00 | 227.33 MB | -| BinaryTrees_2 | NativeAOT 7.0 | 90.32 ms | 0.96 | 227.33 MB | - - -Some of .NET features are not supported by Native AOT, that is why you may want to filter them out using new `[AotFilter]` attribute: - -```cs -[AotFilter("Currently not supported due to missing metadata.")] -public class Xml_FromStream -``` - -## New features and improvements - -### Hiding Columns - -In [#1621](https://github.com/dotnet/BenchmarkDotNet/pull/1621) [@marcnet80](https://github.com/marcnet80) has reduced the number of columns displayed when multiple runtimes are being compared. - -![](https://user-images.githubusercontent.com/6011991/99952390-defab180-2d7f-11eb-851c-ffef6fbb0647.png#mid) - -In [#1890](https://github.com/dotnet/BenchmarkDotNet/pull/1890) [@YegorStepanov](https://github.com/YegorStepanov) has implemented a set of new APIs that allow for hiding columns. It's also exposed via `-h` and `--hide` command line arguments. - -```cs -[MemoryDiagnoser] // adds Gen0, Gen1, Gen2 and Allocated Bytes columns -[HideColumns(Column.Gen0, Column.Gen1, Column.Gen2)] // dont display GenX columns -public class IntroHidingColumns -{ - [Benchmark] - public byte[] AllocateArray() => new byte[100_000]; -} -``` - -Sample results without `[HideColumns]`: - - -| Method | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|-------------- |---------:|----------:|----------:|--------:|--------:|--------:|----------:| -| AllocateArray | 3.303 us | 0.0465 us | 0.0435 us | 31.2462 | 31.2462 | 31.2462 | 97.69 KB | - -With: - -| Method | Mean | Error | StdDev | Allocated | -|-------------- |---------:|----------:|----------:|----------:| -| AllocateArray | 3.489 us | 0.0662 us | 0.0763 us | 97.69 KB | - -Imagine how much time [@YegorStepanov](https://github.com/YegorStepanov) has saved to all the people who so far were removing the columns manually from the results before publishing them on GitHub! - -### Allocation Ratio Column - -In [#1859](https://github.com/dotnet/BenchmarkDotNet/pull/1859) [@YegorStepanov](https://github.com/YegorStepanov) has added Allocation Ratio Column. It's enabled by default when `MemoryDiagnoser` is used and one of the benchmarks is marked as `[Benchmark(Baseline = true)]` or when there are multuple jobs defined and one of them is marked as baseline. - -```cs -[MemoryDiagnoser] -public class AllocationColumnSample -{ - [Benchmark(Baseline = true)] - [Arguments("test")] - public string Builder(string value) - { - StringBuilder sb = new (value); - - for (int i = 0; i < 10; i++) - sb.Append(value); - - return sb.ToString(); - } - - [Benchmark] - [Arguments("test")] - public string Concatenation(string value) - { - string result = value; - - for (int i = 0; i < 10; i++) - result += value; - - return result; - } -} -``` - -| Method | value | Mean | Error | StdDev | Ratio | Gen 0 | Allocated | Alloc Ratio | -|-------------- |------ |---------:|--------:|--------:|------:|-------:|----------:|------------:| -| Builder | test | 127.9 ns | 0.49 ns | 0.43 ns | 1.00 | 0.0544 | 456 B | 1.00 | -| Concatenation | test | 120.2 ns | 0.94 ns | 0.88 ns | 0.94 | 0.0908 | 760 B | 1.67 | - -### Progress and estimated finish time - -In [#1909](https://github.com/dotnet/BenchmarkDotNet/pull/1909) [@adamsitnik](https://github.com/adamsitnik) has added logging of progress and estimated finish time. - -```log -// ** Remained 5211 (99.9%) benchmark(s) to run. Estimated finish 2022-08-25 22:26 (9h 7m from now) ** -``` - -### arm64 support for BenchmarkDotNet.Diagnostics.Windows package - -Due to the [update](https://github.com/dotnet/BenchmarkDotNet/pull/2030) to [TraceEvent 3.0](https://www.nuget.org/packages/Microsoft.Diagnostics.Tracing.TraceEvent) `BenchmarkDotNet.Diagnostics.Windows` package has now arm64 support. Which means that you can use `EtwProfiler` and other ETW-based diagnosers on Windows arm64. - -It would not be possible without [@brianrob](https://github.com/brianrob) who implemented arm64 support for TraceEvent in [#1533](https://github.com/microsoft/perfview/pull/1533)! - -### Hardware Intrinsics information - -In [#2051](https://github.com/dotnet/BenchmarkDotNet/pull/2051) [@adamsitnik](https://github.com/adamsitnik) has extended the hardware information printed in the Summary table with Hardware Intrinsics information. - -![](https://user-images.githubusercontent.com/6011991/180951306-bf8c9f8f-469a-4d7c-8ee1-d7fa47acc0e1.png#mid) - -Sine the space in Summary table is quite limited, we full information is printed only in the log: - -![](https://user-images.githubusercontent.com/6011991/180951531-38c6b6d0-ba6f-4766-a305-b2e306e8420b.png#mid) - -Special thanks to [@tannergooding](https://github.com/tannergooding) who provided a lot of [very valuable feedback](https://github.com/dotnet/BenchmarkDotNet/pull/2051#issuecomment-1194368152) and [@MichalPetryka](https://github.com/MichalPetryka) who contributed an improvement [#2066](https://github.com/dotnet/BenchmarkDotNet/pull/2066) for older runtimes. - -### Other improvements - -* WASM toolchain has received a lot of improvements from various .NET Team members: [#1769](https://github.com/dotnet/BenchmarkDotNet/pull/1769), [#1936](https://github.com/dotnet/BenchmarkDotNet/pull/1936), [#1938](https://github.com/dotnet/BenchmarkDotNet/pull/1938), [#1982](https://github.com/dotnet/BenchmarkDotNet/pull/1982). -* Dependencies and TFMs updates: [#1805](https://github.com/dotnet/BenchmarkDotNet/pull/1805), [#1978](https://github.com/dotnet/BenchmarkDotNet/pull/1978), [#2012](https://github.com/dotnet/BenchmarkDotNet/pull/2012), [#2019](https://github.com/dotnet/BenchmarkDotNet/pull/2019), [#2035](https://github.com/dotnet/BenchmarkDotNet/pull/2035). -* Ensure proper SummaryStyle handling implemented by [@mawosoft](https://github.com/mawosoft) in [#1828](https://github.com/dotnet/BenchmarkDotNet/pull/1828). -* Preserving `EnablePreviewFeatures` project setting which gives the possibility to benchmark preview .NET features. Implemented by [@kkokosa](https://github.com/kkokosa) in [#1842](https://github.com/dotnet/BenchmarkDotNet/pull/1842). -* CI: Using non-deprecated macOS pool on Azure Pipelines, implemented by [@akoeplinger](https://github.com/akoeplinger) in [#1847](https://github.com/dotnet/BenchmarkDotNet/pull/1847) -* CI: Updating Cake to 2.0.0, adopting frosting project style. Implemented by [@AndreyAkinshin](https://github.com/AndreyAkinshin) in [#1865](https://github.com/dotnet/BenchmarkDotNet/pull/1865). -* Detecting ReSharper's Dynamic Program Analysis. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1874](https://github.com/dotnet/BenchmarkDotNet/pull/1874). -* Preventing benchmark failure when some of the exporters fail. Implemented by [@epeshk](https://github.com/epeshk) in [#1902](https://github.com/dotnet/BenchmarkDotNet/pull/1902). -* Don't use the diagnosers when benchmarking has failed. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1903](https://github.com/dotnet/BenchmarkDotNet/pull/1903). -* Ensuring the default order of benchmarks is the same as declared in source code. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1907](https://github.com/dotnet/BenchmarkDotNet/pull/1907). -* Making `BuildTimeout` configurable. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1906](https://github.com/dotnet/BenchmarkDotNet/pull/1906). -* Notify users about private methods with Setup/Cleanup attributes. Implemented by [@epeshk](https://github.com/epeshk) in [#1912](https://github.com/dotnet/BenchmarkDotNet/pull/1912). -* Don't run Roslyn Analyzers for the generated code. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1917](https://github.com/dotnet/BenchmarkDotNet/pull/1917). -* Ensure `WorkloadActionUnroll` and similar are optimized if possible. Implemented by [@AndyAyersMS](https://github.com/AndyAyersMS) in [#1935](https://github.com/dotnet/BenchmarkDotNet/pull/1935). -* Don't use blocking acknowledgments when there is no need to. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1940](https://github.com/dotnet/BenchmarkDotNet/pull/1940). -* Executor: Don't use Process.ExitCode, unless the process has exited. Implemented by [@radical](https://github.com/radical) in [#1947](https://github.com/dotnet/BenchmarkDotNet/pull/1947). -* Revise heuristic for initial jitting. Implemented by [@AndyAyersMS](https://github.com/AndyAyersMS) in [#1949](https://github.com/dotnet/BenchmarkDotNet/pull/1949). -* Allow logging build commands output. Implemented by [@radical](https://github.com/radical) in [#1950](https://github.com/dotnet/BenchmarkDotNet/pull/1950). -* Change Mono AOT mode to Normal AOT with LLVM JIT fall back. Implemented by [@fanyang-mono](https://github.com/fanyang-mono) in [#1990](https://github.com/dotnet/BenchmarkDotNet/pull/1990). - -### Glob filters support for DisassemblyDiagnoser - -So far, the disassembler was always loading the type that was generated by BDN, searching for the benchmark method, disassembling it and when encountered direct method calls, disassembling the called methods as well (if their depth was lesser or equal to max configured depth). - -This was working fine, but only for direct method calls. For indirect, the disassembly was incomplete. - -In [#2072](https://github.com/dotnet/BenchmarkDotNet/pull/2072) [@adamsitnik](https://github.com/adamsitnik) has added the possibility to filter methods disassembled by the `DisassemblyDiagnoser`. - -The users can now pass `--disasmFilter $globPattern` and it's going to be applied to full signatures of all methods available for disassembling. Examples: -* `--disasmFilter *System.Text*` - disassemble all `System.Text` methods. -* `--disasmFilter *` - disassemble all possible methods. - -Moreover, [ClrMD](https://github.com/microsoft/clrmd) was updated to v2 ([#2040](https://github.com/dotnet/BenchmarkDotNet/pull/2040)) and few disassembler bugs have been fixed ([#2075](https://github.com/dotnet/BenchmarkDotNet/pull/2075), [#2078](https://github.com/dotnet/BenchmarkDotNet/pull/2078)). We are expecting that the disassembler will be more reliable now. - -### Docs and Samples improvements - -Big thanks to [@SnakyBeaky](https://github.com/SnakyBeaky), [@Distinctlyminty](https://github.com/Distinctlyminty), [@asaf92](https://github.com/asaf92), [@adamsitnik](https://github.com/adamsitnik) and [@eiriktsarpalis](https://github.com/eiriktsarpalis) who have improved our docs, samples and error messages! - -[#1776](https://github.com/dotnet/BenchmarkDotNet/pull/1776), [#1797](https://github.com/dotnet/BenchmarkDotNet/pull/1797), [#1850](https://github.com/dotnet/BenchmarkDotNet/pull/1850), [#1861](https://github.com/dotnet/BenchmarkDotNet/pull/1861), [#1939](https://github.com/dotnet/BenchmarkDotNet/pull/1939), [#1974](https://github.com/dotnet/BenchmarkDotNet/pull/1974), [#1997](https://github.com/dotnet/BenchmarkDotNet/pull/1997), [#2042](https://github.com/dotnet/BenchmarkDotNet/pull/2042), [#2050](https://github.com/dotnet/BenchmarkDotNet/pull/2050), [#2068](https://github.com/dotnet/BenchmarkDotNet/pull/2068). - -## Bug fixes - -* WASM: [#1811](https://github.com/dotnet/BenchmarkDotNet/pull/1811), [#1846](https://github.com/dotnet/BenchmarkDotNet/pull/1846), [#1916](https://github.com/dotnet/BenchmarkDotNet/pull/1916), [#1926](https://github.com/dotnet/BenchmarkDotNet/pull/1926), [#1932](https://github.com/dotnet/BenchmarkDotNet/pull/1932). -* Diagnoser-provided Analysers weren't automatically added to Config. Fixed by [@mawosoft](https://github.com/mawosoft) in [#1790](https://github.com/dotnet/BenchmarkDotNet/pull/1790). -* Exportes could been duplicated. Fixed by [@workgroupengineering](https://github.com/workgroupengineering) in [#1796](https://github.com/dotnet/BenchmarkDotNet/pull/1796). -* Small bug in SummaryStyle. Fixed by [@mawosoft](https://github.com/mawosoft) in [#1801](https://github.com/dotnet/BenchmarkDotNet/pull/1801). -* `InvalidOperationException/NullReferenceException` in `SmartParaemter`. Fixed by [@mawosoft](https://github.com/mawosoft) in [#1810](https://github.com/dotnet/BenchmarkDotNet/pull/1810). -* Failures caused by colons in benchmark name. Fixed by [@ronbrogan](https://github.com/ronbrogan) in [#1823](https://github.com/dotnet/BenchmarkDotNet/pull/1823). -* Some benchmark arugments were not properly escaped and were causing process launcher to crush. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1841](https://github.com/dotnet/BenchmarkDotNet/pull/1841) -* Invalid size specifiers for Memory and Disassembly diagnosers. Fixed by [@YegorStepanov](https://github.com/YegorStepanov) in [#1854](https://github.com/dotnet/BenchmarkDotNet/pull/1854) and [#1855](https://github.com/dotnet/BenchmarkDotNet/pull/1855). -* Respect LogicalGroup order in DefaultOrderer. Fixed by [@AndreyAkinshin](https://github.com/AndreyAkinshin) in [#1866](https://github.com/dotnet/BenchmarkDotNet/pull/1866). -* Endless loop in user interaction with redirected input. Fixed by [@tmds](https://github.com/tmds) in [#](https://github.com/dotnet/BenchmarkDotNet/pull/1870). -* Broken power plan support. Fixed by [@YegorStepanov](https://github.com/YegorStepanov) in [#1885](https://github.com/dotnet/BenchmarkDotNet/pull/1885). -* `BytesAllocatedPerOperation` was not being output by the JSON and XML exporters. Fixed by [#martincostello](https://github.com/martincostello) in [#1919](https://github.com/dotnet/BenchmarkDotNet/pull/1919). -* Incorrect default InvocationCount in the summary table. Fixed by [@AndreyAkinshin](https://github.com/AndreyAkinshin) in [#1929](https://github.com/dotnet/BenchmarkDotNet/issues/1929). -* Failed build output was printed in reverse order. Fixed by [@radical](https://github.com/radical) in [#1945](https://github.com/dotnet/BenchmarkDotNet/pull/1945). -* Build failures due to `NETSDK1150`. Fixed by [@OlegOLK](https://github.com/OlegOLK) in [#1981](https://github.com/dotnet/BenchmarkDotNet/pull/1981). -* `MetricCoumn` was not respecting provided units when formatting values. Fixed by [@mawosoft](https://github.com/mawosoft) in [#2033](https://github.com/dotnet/BenchmarkDotNet/pull/2033). -* Generating invalid code that was causing benchmark failures. Fixed by [@mawosoft](https://github.com/mawosoft) in [#2041](https://github.com/dotnet/BenchmarkDotNet/pull/2041). -* CI: non-master build branches were publishing artifacts to the CI feed. Fixed by [@mawosoft](https://github.com/mawosoft) in [#2047](https://github.com/dotnet/BenchmarkDotNet/pull/2047). -* Comments in the project files were causing build failures. Fixed by [@mawosoft](https://github.com/mawosoft) in [#2056](https://github.com/dotnet/BenchmarkDotNet/pull/2056). diff --git a/docs/_changelog/header/v0.13.3.md b/docs/_changelog/header/v0.13.3.md deleted file mode 100644 index db1514edf6..0000000000 --- a/docs/_changelog/header/v0.13.3.md +++ /dev/null @@ -1,91 +0,0 @@ -## Highlights - -* New supported technologies - * Add arm64 disassembler - [#1422](https://github.com/dotnet/BenchmarkDotNet/issues/1422) - [#2127](https://github.com/dotnet/BenchmarkDotNet/pull/2127) - [#2107](https://github.com/dotnet/BenchmarkDotNet/pull/2107) - [#2123](https://github.com/dotnet/BenchmarkDotNet/pull/2123) - [#2070](https://github.com/dotnet/BenchmarkDotNet/issues/2070) - [#2118](https://github.com/dotnet/BenchmarkDotNet/pull/2118) - [#2119](https://github.com/dotnet/BenchmarkDotNet/pull/2119) - [#2234](https://github.com/dotnet/BenchmarkDotNet/pull/2234) - [#2222](https://github.com/dotnet/BenchmarkDotNet/pull/2222) - [#2212](https://github.com/dotnet/BenchmarkDotNet/pull/2212) - [9ee1/Capstone.NET#37](https://github.com/9ee1/Capstone.NET/pull/37) - * Initial .NET 8 support - [#2192](https://github.com/dotnet/BenchmarkDotNet/pull/2192) - * .NET 6/7 MonoVM support - [#2064](https://github.com/dotnet/BenchmarkDotNet/issues/2064) - [#2142](https://github.com/dotnet/BenchmarkDotNet/pull/2142) - [#2227](https://github.com/dotnet/BenchmarkDotNet/pull/2227) - [#2230](https://github.com/dotnet/BenchmarkDotNet/pull/2230) - * Armv6 and Ppc64le architectures support - [#2216](https://github.com/dotnet/BenchmarkDotNet/issues/2216) - [#2219](https://github.com/dotnet/BenchmarkDotNet/pull/2219) -* Improved support - * Improved WASM support - [#2201](https://github.com/dotnet/BenchmarkDotNet/pull/2201) - [#2099](https://github.com/dotnet/BenchmarkDotNet/issues/2099) - [#2154](https://github.com/dotnet/BenchmarkDotNet/pull/2154) - [#2112](https://github.com/dotnet/BenchmarkDotNet/pull/2112) - * Improved NativeAOT support - [#2095](https://github.com/dotnet/BenchmarkDotNet/pull/2095) - [#2221](https://github.com/dotnet/BenchmarkDotNet/pull/2221) - * Improved Android support - [#2231](https://github.com/dotnet/BenchmarkDotNet/pull/2231) - * 32-bit benchmarks can now handle addresses larger than 2GB with the help of `LargeAddressAware` - [#1469](https://github.com/dotnet/BenchmarkDotNet/issues/1469) - [#2145](https://github.com/dotnet/BenchmarkDotNet/pull/2145) - * Support 64bit affinity masks - [#2211](https://github.com/dotnet/BenchmarkDotNet/issues/2211) - [#2228](https://github.com/dotnet/BenchmarkDotNet/pull/2228) -* New features - * Add `ExceptionDiagnoser` - [#1736](https://github.com/dotnet/BenchmarkDotNet/issues/1736) - [#2169](https://github.com/dotnet/BenchmarkDotNet/pull/2169) - [#2182](https://github.com/dotnet/BenchmarkDotNet/pull/2182) - * Add `PerfCollectProfiler` - [#2117](https://github.com/dotnet/BenchmarkDotNet/pull/2117) - * Incremental benchmark execution with the help of `--resume` - [#1799](https://github.com/dotnet/BenchmarkDotNet/issues/1799) - [#2164](https://github.com/dotnet/BenchmarkDotNet/pull/2164) - * Taskbar progress - [#2102](https://github.com/dotnet/BenchmarkDotNet/issues/2102) - [#2158](https://github.com/dotnet/BenchmarkDotNet/pull/2158) - [#2140](https://github.com/dotnet/BenchmarkDotNet/pull/2140) - * Support `--noForcedGCs` to avoid forced GC between benchmark runs - [#2101](https://github.com/dotnet/BenchmarkDotNet/pull/2101) - * Added apples to apples comparison mode - [#2116](https://github.com/dotnet/BenchmarkDotNet/pull/2116) - [#2193](https://github.com/dotnet/BenchmarkDotNet/pull/2193) - * Communication between the host process and the benchmark process is now using pipes instead of output parsing - [#2092](https://github.com/dotnet/BenchmarkDotNet/pull/2092) - [#1933](https://github.com/dotnet/BenchmarkDotNet/issues/1933) - [#2189](https://github.com/dotnet/BenchmarkDotNet/issues/2189) - [#2207](https://github.com/dotnet/BenchmarkDotNet/pull/2207) - [#2200](https://github.com/dotnet/BenchmarkDotNet/pull/2200) -* Dozens of bugfixes - -## Special Thanks - -We would like to highlight some important contributors who helped us with this release: - -1. [OpenHack'22 (devexperts.com)](https://code.devexperts.com/event/openhack22) hackathon sponsored by the DevExperts company. - As part of this hackathon we have received following PRs: - * [#2132](https://github.com/dotnet/BenchmarkDotNet/pull/2132) fix: include argument and param names in --filter (by [@blouflashdb](https://github.com/blouflashdb)) - * [#2140](https://github.com/dotnet/BenchmarkDotNet/pull/2140) Update console title with benchmark information (by [@franciscomoloureiro](https://github.com/franciscomoloureiro)) - * [#2142](https://github.com/dotnet/BenchmarkDotNet/pull/2142) Issue 2064: Mono70 moniker (by [@Serg046](https://github.com/Serg046)) - * [#2148](https://github.com/dotnet/BenchmarkDotNet/pull/2148) adding validation errors when the benchmarks are unsupported (by [@emanuel-v-r](https://github.com/emanuel-v-r)) - * [#2160](https://github.com/dotnet/BenchmarkDotNet/pull/2160) Corrected logic to restore foreground color in ConsoleLogger.cs (by [@farQtech](https://github.com/farQtech)) - * [#2164](https://github.com/dotnet/BenchmarkDotNet/pull/2164) 1799 adding resume arg (by [@melias](https://github.com/melias)) - * [#2169](https://github.com/dotnet/BenchmarkDotNet/pull/2169) Issue #1736: Add ExceptionDiagnoser (by [@Serg046](https://github.com/Serg046)) - * [#2161](https://github.com/dotnet/BenchmarkDotNet/pull/2161) add quiet logger (by [@franciscomoloureiro](https://github.com/franciscomoloureiro)) **(not merged yet)** - * [#2171](https://github.com/dotnet/BenchmarkDotNet/pull/2171) Issue #1024: Calculate baseline by the fastest benchmark (by [@Serg046](https://github.com/Serg046)) **(not merged yet)** - -2. Jan Vorlicek helped to implement arm64 disassembler during an internal Microsoft Hackathon: - * [#2107](https://github.com/dotnet/BenchmarkDotNet/pull/2107) Implement TryGetReferencedAddress for relative branches (by [@janvorli](https://github.com/janvorli)) - * [#2123](https://github.com/dotnet/BenchmarkDotNet/pull/2123) Added other arm64 constant form extraction plus other changes (by [@janvorli](https://github.com/janvorli)) - -3. Ahmed Garhy (maintainer of Capstone.NET) helped to improve Capstone.NET, which was need to implement arm64 disassembler: - * [9ee1/Capstone.NET#37](https://github.com/9ee1/Capstone.NET/pull/37) Sign Assembly with a Strong Name (by [@9ee1](https://github.com/9ee1)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.4.md b/docs/_changelog/header/v0.13.4.md deleted file mode 100644 index 3ae8407ee4..0000000000 --- a/docs/_changelog/header/v0.13.4.md +++ /dev/null @@ -1,26 +0,0 @@ -## Highlights - -* Fixed LINQPad support - [#2237](https://github.com/dotnet/BenchmarkDotNet/issues/2237) - [#2238](https://github.com/dotnet/BenchmarkDotNet/pull/2238) -* New `JitStatsDiagnoser` - [#2243](https://github.com/dotnet/BenchmarkDotNet/pull/2243) -* Minor documentation improvements - [#2206](https://github.com/dotnet/BenchmarkDotNet/pull/2206) - [#2218](https://github.com/dotnet/BenchmarkDotNet/pull/2218) - -## JitStatsDiagnoser - -This new diagnoser introduced in ([#2243](https://github.com/dotnet/BenchmarkDotNet/pull/2243)) allows getting advanced JIT statistics. - -Sample usage: - -```cmd -dotnet run -c Release -f net7.0 --filter *IntroBasic.Sleep --profiler jit -``` - -Result: - -| Method | Mean | Error | StdDev | Methods JITted | Methods Tiered | JIT allocated memory | -|------- |---------:|---------:|---------:|---------------:|---------------:|---------------------:| -| Sleep | 15.53 ms | 0.034 ms | 0.032 ms | 1,102 | 15 | 221,736 B | \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.5.md b/docs/_changelog/header/v0.13.5.md deleted file mode 100644 index c3915a4f6d..0000000000 --- a/docs/_changelog/header/v0.13.5.md +++ /dev/null @@ -1,34 +0,0 @@ -## Highlights - -* Improved `JitStatsDiagnoser`. - This diagnoser was added in v0.13.4, it shows various stats from the JIT compiler that were collected during entire benchmark run - (amount of JITted methods, - amount of [tiered methods](https://learn.microsoft.com/en-us/dotnet/core/whats-new/dotnet-core-3-0#tiered-compilation), - how much memory JIT allocated during the benchmark). - In this release, we improved metric collection - ([#2246](https://github.com/dotnet/BenchmarkDotNet/pull/2246), - [e715d5](https://github.com/dotnet/BenchmarkDotNet/commit/e715d5bb63984fca65120d9a497f7d16395f9e5b)) - and added the `[JitStatsDiagnoser]` attribute - ([#2250](https://github.com/dotnet/BenchmarkDotNet/pull/2250) - [512413](https://github.com/dotnet/BenchmarkDotNet/commit/512413ceb24077154bdf6d6306138accffe64c7a)). -* Enable strong-named assemblies in the released NuGet packages - [#2258](https://github.com/dotnet/BenchmarkDotNet/issues/2258) - [#2263](https://github.com/dotnet/BenchmarkDotNet/pull/2263) - [5cd288](https://github.com/dotnet/BenchmarkDotNet/commit/5cd288996ca13292fcf638be299c097a600aea7b) -* Avoid keeping referenced values returned from a benchmark in memory - [#1942](https://github.com/dotnet/BenchmarkDotNet/issues/1942) - [#2191](https://github.com/dotnet/BenchmarkDotNet/pull/2191) - [ff5dbe](https://github.com/dotnet/BenchmarkDotNet/commit/ff5dbe662478f547e4be8d734eaeb6a106f40875) -* Keep generated files when MSBuild bin log is requested - [#2252](https://github.com/dotnet/BenchmarkDotNet/issues/2252) - [#2254](https://github.com/dotnet/BenchmarkDotNet/pull/2254) - [d3fbc0](https://github.com/dotnet/BenchmarkDotNet/commit/d3fbc03d6dabeb52f23c6b7e50287150e66957cc) -* Add `Id` for `UnresolvedDiagnoser` (an exception fix) - [#2251](https://github.com/dotnet/BenchmarkDotNet/pull/2251) - [a992b5](https://github.com/dotnet/BenchmarkDotNet/commit/a992b57490e844acf587bc2e01b08a7040dbc8e2) -* Add brand names for Windows 22H2 and macOS 13 - [86f212](https://github.com/dotnet/BenchmarkDotNet/commit/86f212b79e297d87d3942e4c50130fe6e214f3c8) - [0c2699](https://github.com/dotnet/BenchmarkDotNet/commit/0c26996ea685a99068aca71e7ae547b0851d3c64) -* Remove deprecated `InProcessToolchain` - [#2248](https://github.com/dotnet/BenchmarkDotNet/pull/2248) - [615384](https://github.com/dotnet/BenchmarkDotNet/commit/615384d2553434d7f35c03ab3174d761f82c6c2d) diff --git a/docs/_changelog/header/v0.13.6.md b/docs/_changelog/header/v0.13.6.md deleted file mode 100644 index 7667d404ce..0000000000 --- a/docs/_changelog/header/v0.13.6.md +++ /dev/null @@ -1,23 +0,0 @@ -## Highlights - -* New [BenchmarkDotNet.Diagnostics.dotTrace](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace) NuGet package. - Once this package is installed, you can annotate your benchmarks with the `[DotTraceDiagnoser]` and get a [dotTrace](https://www.jetbrains.com/profiler/) performance snapshot at the end of the benchmark run. - [#2328](https://github.com/dotnet/BenchmarkDotNet/pull/2328) -* Updated documentation website. - We migrated to [docfx](https://dotnet.github.io/docfx/) 2.67 and got the refreshed modern template based on bootstrap 5 with dark/light theme switcher. -* Updated [BenchmarkDotNet.Templates](https://www.nuget.org/packages/BenchmarkDotNet.Templates). - Multiple issues were resolved, now you can create new benchmark projects from terminal or your favorite IDE. - [#1658](https://github.com/dotnet/BenchmarkDotNet/issues/1658) - [#1881](https://github.com/dotnet/BenchmarkDotNet/issues/1881) - [#2149](https://github.com/dotnet/BenchmarkDotNet/issues/2149) - [#2338](https://github.com/dotnet/BenchmarkDotNet/pull/2338) -* Response file support. - Now it's possible to pass additional arguments to BenchmarkDotNet using `@filename` syntax. - [#2320](https://github.com/dotnet/BenchmarkDotNet/pull/2320) - [#2348](https://github.com/dotnet/BenchmarkDotNet/pull/2348) -* Custom runtime support. - [#2285](https://github.com/dotnet/BenchmarkDotNet/pull/2285) -* Introduce CategoryDiscoverer, see [`IntroCategoryDiscoverer`](xref:BenchmarkDotNet.Samples.IntroCategoryDiscoverer). - [#2306](https://github.com/dotnet/BenchmarkDotNet/issues/2306) - [#2307](https://github.com/dotnet/BenchmarkDotNet/pull/2307) -* Multiple bug fixes. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.7.md b/docs/_changelog/header/v0.13.7.md deleted file mode 100644 index 5a8487b639..0000000000 --- a/docs/_changelog/header/v0.13.7.md +++ /dev/null @@ -1,3 +0,0 @@ -## Highlights - -This release contains important bug fixes listed below. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.8.md b/docs/_changelog/header/v0.13.8.md deleted file mode 100644 index b9222853e2..0000000000 --- a/docs/_changelog/header/v0.13.8.md +++ /dev/null @@ -1,3 +0,0 @@ -## Highlights - -This release contains important bug fixes. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.9.md b/docs/_changelog/header/v0.13.9.md deleted file mode 100644 index 07e4813c9f..0000000000 --- a/docs/_changelog/header/v0.13.9.md +++ /dev/null @@ -1,3 +0,0 @@ -## Highlights - -This release contains bug fixes. \ No newline at end of file diff --git a/docs/_changelog/header/v0.8.2.md b/docs/_changelog/header/v0.8.2.md deleted file mode 100644 index b06d5ec1c2..0000000000 --- a/docs/_changelog/header/v0.8.2.md +++ /dev/null @@ -1,19 +0,0 @@ -* Changes in the Summary table - * Summary table now supports ResultExtenders that can add new column to the table - * Now we use [StandardError](https://en.wikipedia.org/wiki/Standard_error) (aka `Error`) as the main accuracy metric - * Columns `op/s`, `StdDev` are disabled by default (you can add it via ResultExtenders) -* Statistic improvements, now you have detailed statistic in the console log like follows: -``` -Mean = 118.5298 us, StdError = 1.2863 us (N = 30, StdDev = 7.0454 us) -Min = 109.1602 us, Median = 117.1794 us, Max = 132.5764 us -IQR = 10.1244 us, LowerFence = 98.0834 us, UpperFence = 138.5810 us -ConfidenceInterval = [116.0086 us; 121.0510 us] (CI 95%) -``` -* Added the `Baseline` feature, see [#64](https://github.com/PerfDotNet/BenchmarkDotNet/issues/64) -* Export improvements, now you have files `-report-github.md`, `-report-stackoverflow.md` for easy publishing results on GitHub and StackOverflow. -* Basic plotting. Added new `BenchmarkRPlotExporter` that creates `BuildPlots.R` in the bin directory. It is an R script that generates boxplot and barplot for your benchmarks (you should have installed R with defined `R_HOME` environment variable) -* Updated environment info - * Added Stopwatch `Frequency` and `Resolution` - * Split common benchmark properties (like `Mode`, `Platform`, `Runtime`) in several lines (3 properties per line) -* Log improvements: add total time, statistics, list of exported files -* Bug fixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.0.md b/docs/_changelog/header/v0.9.0.md deleted file mode 100644 index 3d14c4b30c..0000000000 --- a/docs/_changelog/header/v0.9.0.md +++ /dev/null @@ -1,2 +0,0 @@ -* New API -* Autodetermination of amount iteration for warmup/target idle/main iterations, duration of iteration, amount of CLR launches. \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.2.md b/docs/_changelog/header/v0.9.2.md deleted file mode 100644 index 44d8bb5fe0..0000000000 --- a/docs/_changelog/header/v0.9.2.md +++ /dev/null @@ -1 +0,0 @@ -* Dnx451 support (Closed [#51](https://github.com/PerfDotNet/BenchmarkDotNet/issues/51), Merged [#87](https://github.com/PerfDotNet/BenchmarkDotNet/issues/87)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.3.md b/docs/_changelog/header/v0.9.3.md deleted file mode 100644 index f7614fbe00..0000000000 --- a/docs/_changelog/header/v0.9.3.md +++ /dev/null @@ -1 +0,0 @@ -* CoreCLR support (Closed [#52](https://github.com/PerfDotNet/BenchmarkDotNet/issues/52), Merged [#113](https://github.com/PerfDotNet/BenchmarkDotNet/issues/113)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.4.md b/docs/_changelog/header/v0.9.4.md deleted file mode 100644 index 63c47b49d0..0000000000 --- a/docs/_changelog/header/v0.9.4.md +++ /dev/null @@ -1,9 +0,0 @@ -* Improved messages about error in benchmarks, see [#104](https://github.com/PerfDotNet/BenchmarkDotNet/issues/104) -* Natural sort order, see [#92](https://github.com/PerfDotNet/BenchmarkDotNet/issues/92), [#95](https://github.com/PerfDotNet/BenchmarkDotNet/issues/95), [#97](https://github.com/PerfDotNet/BenchmarkDotNet/issues/97) -* Improved `double`/`float`/`decimal`/`enum` support for Params, see [#96](https://github.com/PerfDotNet/BenchmarkDotNet/issues/96), [#105](https://github.com/PerfDotNet/BenchmarkDotNet/issues/105), [#116](https://github.com/PerfDotNet/BenchmarkDotNet/issues/116) -* Now environment info includes information about `HardwareTimerKind` and `JitModules` -* Added `DryConfig` -* Improved export performance, closed [#119](https://github.com/PerfDotNet/BenchmarkDotNet/issues/119), merged [#124](https://github.com/PerfDotNet/BenchmarkDotNet/pull/124) -* Better cmd-line discoverability (see [#78](https://github.com/PerfDotNet/BenchmarkDotNet/issues/78)), e.g. run `Benchmark.exe --help` and some useful information will be printed -* Supporting all kinds of references for generated project (exact version, custom paths, GAC, sub-folders, dependent assemblies not copied), closed [#41](https://github.com/PerfDotNet/BenchmarkDotNet/issues/41), [#49](https://github.com/PerfDotNet/BenchmarkDotNet/issues/49), [#72](https://github.com/PerfDotNet/BenchmarkDotNet/issues/72), [#123](https://github.com/PerfDotNet/BenchmarkDotNet/issues/123), merged [#125](https://github.com/PerfDotNet/BenchmarkDotNet/pull/125) -* Friendliness to LinqPad restored, closed [#66](https://github.com/PerfDotNet/BenchmarkDotNet/issues/66), merged [#125](https://github.com/PerfDotNet/BenchmarkDotNet/pull/125) \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.5.md b/docs/_changelog/header/v0.9.5.md deleted file mode 100644 index 1d429dcc35..0000000000 --- a/docs/_changelog/header/v0.9.5.md +++ /dev/null @@ -1,11 +0,0 @@ -* Added validators, JitOptimizationsValidator detects all non-optimzied dlls that were referenced [#134](https://github.com/PerfDotNet/BenchmarkDotNet/issues/134) -* **Strong naming** [#101](https://github.com/PerfDotNet/BenchmarkDotNet/issues/101) -* Add `IOrderProvider` [#107](https://github.com/PerfDotNet/BenchmarkDotNet/issues/107) -* **Putting all the generated artifacts in a separate folder: ./BenchmarkDotNet.Artifacts/results** and ./BenchmarkDotNet.Artifacts/bin [#94](https://github.com/PerfDotNet/BenchmarkDotNet/issues/94) -* Printing dotnet cli version for .NET Core and Dnx451, informing user when not installed. Closed [#128](https://github.com/PerfDotNet/BenchmarkDotNet/issues/128) -* Supporting assembly redirects [#67](https://github.com/PerfDotNet/BenchmarkDotNet/issues/67) -* Changed used msbuild version: 12 for .NET 4.5 (VS 2013), 14 for .NET 4.6 (VS 2015). Closed [#132](https://github.com/PerfDotNet/BenchmarkDotNet/issues/132) and [#137](https://github.com/PerfDotNet/BenchmarkDotNet/issues/137) -* Switched to new ‘dotnet’ target framework monikers (dotnet5.4 instead of dnxcore50), [why](https://github.com/aspnet/Announcements/issues/98) -* dnx452, dnx46, net462 support added -* Executing single Benchmark for multiple Runtimes also with Diagnoser attached (see [#117](https://github.com/PerfDotNet/BenchmarkDotNet/pull/117)) -* Misc minor changes \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.6.md b/docs/_changelog/header/v0.9.6.md deleted file mode 100644 index b69b6591bd..0000000000 --- a/docs/_changelog/header/v0.9.6.md +++ /dev/null @@ -1,3 +0,0 @@ -* Added Percentiles (see [#164](https://github.com/PerfDotNet/BenchmarkDotNet/pull/164)) -* Added support for Json export (see [#84](https://github.com/PerfDotNet/BenchmarkDotNet/issues/84)) -* Bugfixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.7.md b/docs/_changelog/header/v0.9.7.md deleted file mode 100644 index 560d956baf..0000000000 --- a/docs/_changelog/header/v0.9.7.md +++ /dev/null @@ -1,2 +0,0 @@ -* .NET Core RC2 support (see [#187](https://github.com/PerfDotNet/BenchmarkDotNet/pull/187)) -* Bugfixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.8.md b/docs/_changelog/header/v0.9.8.md deleted file mode 100644 index 055ca19bf6..0000000000 --- a/docs/_changelog/header/v0.9.8.md +++ /dev/null @@ -1,8 +0,0 @@ -* CoreCLR RTM support (see [#216](https://github.com/PerfDotNet/BenchmarkDotNet/issues/216)). **Breaking change:** we have dropped dnx451 and dnxcore50 support. -* Migration from MSBuild to Roslyn, which supports Mono on Linux and MacOS (see [#149](https://github.com/PerfDotNet/BenchmarkDotNet/issues/149)). **Breaking change:** we have dropped .NET 4.0 support. -* Ability to manage GC mode: turn on/off the Server/Concurrent GC modes, extend to CPU groups, set gcAllowVeryLargeObjects and avoid BenchmarkDotNet from forcing GC.Collect (see [#188](https://github.com/PerfDotNet/BenchmarkDotNet/issues/188), [#76](https://github.com/PerfDotNet/BenchmarkDotNet/issues/76) and [#211](https://github.com/PerfDotNet/BenchmarkDotNet/issues/211)) -* Support CopyToOutput (see [#212](https://github.com/PerfDotNet/BenchmarkDotNet/issues/212)). Now you can use native dependencies as well as custom files. -* Copying custom settings from app.config (see [#108](https://github.com/PerfDotNet/BenchmarkDotNet/issues/108)). It means we support assembly binding redirects as well as custom connection strings etc. -* AsciiDocExporter (see [#169](https://github.com/PerfDotNet/BenchmarkDotNet/pull/169)) -* Framework setting in Job has been removed (see [#194](https://github.com/PerfDotNet/BenchmarkDotNet/issues/194)) -* Minor bugfixes and improvements \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.9.md b/docs/_changelog/header/v0.9.9.md deleted file mode 100644 index a3352ca136..0000000000 --- a/docs/_changelog/header/v0.9.9.md +++ /dev/null @@ -1,9 +0,0 @@ -* Attribute config style (see [#166](https://github.com/PerfDotNet/BenchmarkDotNet/issues/166)) -* [Online documentation](https://perfdotnet.github.io/BenchmarkDotNet/index.htm) (see [#219](https://github.com/PerfDotNet/BenchmarkDotNet/issues/219)) -* Mono LLVM support (see [#226](https://github.com/PerfDotNet/BenchmarkDotNet/issues/226)) -* Async method support (see [#236](https://github.com/PerfDotNet/BenchmarkDotNet/issues/236)) -* NuGet packages and repo layout restructuring (see [#225](https://github.com/PerfDotNet/BenchmarkDotNet/issues/225), [#228](https://github.com/PerfDotNet/BenchmarkDotNet/issues/228)) -* `[Cleanup]` attribute (see [#215](https://github.com/PerfDotNet/BenchmarkDotNet/issues/215)) -* New statistics columns: `Skewness`, `Kurtosis`, `WelchTTestPValue`, Improved math for the `Scaled` column -* Now current default branch is `master` -* Minor improvements and bug fixes \ No newline at end of file diff --git a/docs/articles/configs/diagnosers.md b/docs/articles/configs/diagnosers.md index cfb5532a40..e81943310a 100644 --- a/docs/articles/configs/diagnosers.md +++ b/docs/articles/configs/diagnosers.md @@ -35,10 +35,10 @@ The current Diagnosers are: Please see Adam Sitnik's [blog post](https://adamsitnik.com/ConcurrencyVisualizer-Profiler/) for all the details. - Native Memory Profiler (`NativeMemoryProfiler`) It uses `EtwProfiler` to profile the code using ETW and adds the extra columns `Allocated native memory` and `Native memory leak`. - Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.com/2019/08/analyzing-native-memory-allocation-with-benchmarkdotnet/) for all the details. + Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.github.io/2019/08/analyzing-native-memory-allocation-with-benchmarkdotnet/) for all the details. - Event Pipe Profiler (`EventPipeProfiler`). It is a cross-platform profiler that allows profile .NET code on every platform - Windows, Linux, macOS. - Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.com/2020/04/cross-platform-profiling-.net-code-with-benchmarkdotnet/) for all the details. + Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.github.io/2020/04/cross-platform-profiling-.net-code-with-benchmarkdotnet/) for all the details. - Threading Diagnoser (`ThreadingDiagnoser`) - .NET Core 3.0+ diagnoser that reports some Threading statistics. - Exception Diagnoser (`ExceptionDiagnoser`) - a diagnoser that reports the frequency of exceptions thrown during the operation. @@ -59,11 +59,11 @@ private class Config : ManualConfig { public Config() { - Add(MemoryDiagnoser.Default); - Add(new InliningDiagnoser()); - Add(new EtwProfiler()); - Add(ThreadingDiagnoser.Default); - Add(ExceptionDiagnoser.Default); + AddDiagnoser(MemoryDiagnoser.Default); + AddDiagnoser(new InliningDiagnoser()); + AddDiagnoser(new EtwProfiler()); + AddDiagnoser(ThreadingDiagnoser.Default); + AddDiagnoser(ExceptionDiagnoser.Default); } } ``` @@ -86,7 +86,6 @@ In BenchmarkDotNet, 1kB = 1024B, 1MB = 1024kB, and so on. The column Gen X means * In order to not affect main results we perform a separate run if any diagnoser is used. That's why it might take more time to execute benchmarks. * MemoryDiagnoser: - * Mono currently [does not](https://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes-in-mono) expose any api to get the number of allocated bytes. That's why our Mono users will get `?` in Allocated column. * In order to get the number of allocated bytes in cross platform way we are using `GC.GetAllocatedBytesForCurrentThread` which recently got [exposed](https://github.com/dotnet/corefx/pull/12489) for netcoreapp1.1. That's why BenchmarkDotNet does not support netcoreapp1.0 from version 0.10.1. * MemoryDiagnoser is `99.5%` accurate about allocated memory when using default settings or Job.ShortRun (or any longer job than it). * Threading Diagnoser: diff --git a/docs/articles/configs/jobs.md b/docs/articles/configs/jobs.md index a6085e0bdb..ad0cf4559c 100644 --- a/docs/articles/configs/jobs.md +++ b/docs/articles/configs/jobs.md @@ -98,20 +98,20 @@ public class MyBenchmarks { public Config() { - Add( - new Job("MySuperJob", RunMode.Dry, EnvMode.RyuJitX64) + AddJob( + new Job("MySuperJob", RunMode.Dry, EnvironmentMode.RyuJitX64) { - Env = { Runtime = Runtime.Core }, + Environment = { Runtime = CoreRuntime.Core90 }, Run = { LaunchCount = 5, IterationTime = TimeInterval.Millisecond * 200 }, - Accuracy = { MaxStdErrRelative = 0.01 } + Accuracy = { MaxRelativeError = 0.01 } }); // The same, using the .With() factory methods: - Add( + AddJob( Job.Dry .WithPlatform(Platform.X64) .WithJit(Jit.RyuJit) - .WithRuntime(Runtime.Core) + .WithRuntime(CoreRuntime.Core90) .WithLaunchCount(5) .WithIterationTime(TimeInterval.Millisecond * 200) .WithMaxRelativeError(0.01) @@ -122,26 +122,26 @@ public class MyBenchmarks } ``` -Basically, it's a good idea to start with predefined values (e.g. `EnvMode.RyuJitX64` and `RunMode.Dry` passed as constructor args) and modify rest of the properties using property setters or with help of object initializer syntax. +Basically, it's a good idea to start with predefined values (e.g. `EnvironmentMode.RyuJitX64` and `RunMode.Dry` passed as constructor args) and modify rest of the properties using property setters or with help of object initializer syntax. Note that the job cannot be modified after it's added into config. Trying to set a value on property of the frozen job will throw an `InvalidOperationException`. Use the `Job.Frozen` property to determine if the code properties can be altered. If you do want to create a new job based on frozen one (all predefined job values are frozen) you can use the `.With()` extension method ```cs - var newJob = Job.Dry.With(Platform.X64); + var newJob = Job.Dry.WithPlatform(Platform.X64); ``` or pass the frozen value as a constructor argument ```c# - var newJob = new Job(Job.Dry) { Env = { Platform = Platform.X64 } }; + var newJob = new Job(Job.Dry) { Environment = { Platform = Platform.X64 } }; ``` or use the `.Apply()` method on unfrozen job ```c# - var newJob = new Job() { Env = { Platform = Platform.X64 } }.Apply(Job.Dry); + var newJob = new Job() { Environment = { Platform = Platform.X64 } }.Apply(Job.Dry); ``` in any case the Id property will not be transfered and you must pass it explicitly (using the .ctor id argument or the `.WithId()` extension method). @@ -152,7 +152,9 @@ You can also add new jobs via attributes. Examples: ```cs [DryJob] -[ClrJob, CoreJob, MonoJob] +[MonoJob] +[SimpleJob(RuntimeMoniker.Net90)] +[SimpleJob(RuntimeMoniker.NetCoreApp31)] [LegacyJitX86Job, LegacyJitX64Job, RyuJitX64Job] [SimpleJob(RunStrategy.ColdStart, launchCount: 1, warmupCount: 5, iterationCount: 5, id: "FastAndDirtyJob")] public class MyBenchmarkClass @@ -212,7 +214,7 @@ public class MySuperJobAttribute : Attribute, IConfigSource { var job = new Job("MySuperJob", RunMode.Core); job.Env.Platform = Platform.X64; - Config = ManualConfig.CreateEmpty().With(job); + Config = ManualConfig.CreateEmpty().AddJob(job); } public IConfig Config { get; } diff --git a/docs/articles/configs/toolchains.md b/docs/articles/configs/toolchains.md index 8f73bc3784..b47e994a94 100644 --- a/docs/articles/configs/toolchains.md +++ b/docs/articles/configs/toolchains.md @@ -18,14 +18,14 @@ When you run your benchmarks without specifying the toolchain in an explicit way If you want to test multiple frameworks, your project file **MUST target all of them** and you **MUST install the corresponding SDKs**: ```xml -netcoreapp3.0;netcoreapp2.1;net48 +netcoreapp3.1;net8.0;net48 ``` If you run your benchmarks without specifying any custom settings, BenchmarkDotNet is going to run the benchmarks **using the same framework as the host process**: ```cmd -dotnet run -c Release -f netcoreapp2.1 # is going to run the benchmarks using .NET Core 2.1 -dotnet run -c Release -f netcoreapp3.0 # is going to run the benchmarks using .NET Core 3.0 +dotnet run -c Release -f netcoreapp3.1 # is going to run the benchmarks using .NET Core 3.1 +dotnet run -c Release -f net8.0 # is going to run the benchmarks using .NET 8.0 dotnet run -c Release -f net48 # is going to run the benchmarks using .NET 4.8 mono $pathToExe # is going to run the benchmarks using Mono from your PATH ``` @@ -33,8 +33,8 @@ mono $pathToExe # is going to run the benchmarks using Mo To run the benchmarks for multiple runtimes with a single command, you need to specify the target framework moniker names via `--runtimes|-r` console argument: ```cmd -dotnet run -c Release -f netcoreapp2.1 --runtimes netcoreapp2.1 netcoreapp3.0 # is going to run the benchmarks using .NET Core 2.1 and .NET Core 3.0 -dotnet run -c Release -f netcoreapp2.1 --runtimes netcoreapp2.1 net48 # is going to run the benchmarks using .NET Core 2.1 and .NET 4.8 +dotnet run -c Release -f net8.0 --runtimes net8.0 netcoreapp3.1 # is going to run the benchmarks using .NET 8.0 and .NET Core 3.1 +dotnet run -c Release -f net8.0 --runtimes net8.0 net48 # is going to run the benchmarks using .NET 8.0 and .NET 4.8 ``` What is going to happen if you provide multiple Full .NET Framework monikers? Let's say: @@ -67,8 +67,8 @@ namespace BenchmarkDotNet.Samples { [SimpleJob(RuntimeMoniker.Net48)] [SimpleJob(RuntimeMoniker.Mono)] - [SimpleJob(RuntimeMoniker.NetCoreApp21)] - [SimpleJob(RuntimeMoniker.NetCoreApp30)] + [SimpleJob(RuntimeMoniker.NetCoreApp31)] + [SimpleJob(RuntimeMoniker.Net80)] public class TheClassWithBenchmarks ``` @@ -87,10 +87,9 @@ namespace BenchmarkDotNet.Samples static void Main(string[] args) { var config = DefaultConfig.Instance - .With(Job.Default.With(CoreRuntime.Core21)) - .With(Job.Default.With(CoreRuntime.Core30)) - .With(Job.Default.With(ClrRuntime.Net48)) - .With(Job.Default.With(MonoRuntime.Default)); + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80)) + .AddJob(Job.Default.WithRuntime(ClrRuntime.Net48)) + .AddJob(Job.Default.WithRuntime(MonoRuntime.Default)); BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) @@ -115,9 +114,9 @@ public class MyConfig : ManualConfig Add(Job.Default.With( CsProjCoreToolchain.From( new NetCoreAppSettings( - targetFrameworkMoniker: "netcoreapp2.1", - runtimeFrameworkVersion: "2.1.0-preview2-25628-01", - name: ".NET Core 2.1")))); + targetFrameworkMoniker: "net8.0-windows", + runtimeFrameworkVersion: "8.0.101", + name: ".NET 8.0 Windows")))); } } ``` @@ -130,8 +129,8 @@ It's possible to benchmark a private build of .NET Runtime. All you need to do i BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) .Run(args, - DefaultConfig.Instance.With( - Job.ShortRun.With(ClrRuntime.CreateForLocalFullNetFrameworkBuild(version: "4.0")))); + DefaultConfig.Instance.AddJob( + Job.ShortRun.WithRuntime(ClrRuntime.CreateForLocalFullNetFrameworkBuild(version: "4.0")))); ``` This sends the provided version as a `COMPLUS_Version` env var to the benchmarked process. @@ -146,15 +145,15 @@ public class CustomPathsConfig : ManualConfig public CustomPathsConfig() { var dotnetCli32bit = NetCoreAppSettings - .NetCoreApp20 + .NetCoreApp31 .WithCustomDotNetCliPath(@"C:\Program Files (x86)\dotnet\dotnet.exe", "32 bit cli"); var dotnetCli64bit = NetCoreAppSettings - .NetCoreApp20 + .NetCoreApp31 .WithCustomDotNetCliPath(@"C:\Program Files\dotnet\dotnet.exe", "64 bit cli"); - Add(Job.RyuJitX86.With(CsProjCoreToolchain.From(dotnetCli32bit)).WithId("32 bit cli")); - Add(Job.RyuJitX64.With(CsProjCoreToolchain.From(dotnetCli64bit)).WithId("64 bit cli")); + AddJob(Job.RyuJitX86.WithToolchain(CsProjCoreToolchain.From(dotnetCli32bit)).WithId("32 bit cli")); + AddJob(Job.RyuJitX64.WithToolchain(CsProjCoreToolchain.From(dotnetCli64bit)).WithId("64 bit cli")); } } ``` @@ -208,7 +207,7 @@ or: ```cs var config = DefaultConfig.Instance - .With(Job.Default.With(NativeAotRuntime.Net70)); // compiles the benchmarks as net7.0 and uses the latest NativeAOT to build a native app + .AddJob(Job.Default.WithRuntime(NativeAotRuntime.Net70)); // compiles the benchmarks as net7.0 and uses the latest NativeAOT to build a native app BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) @@ -231,8 +230,8 @@ If you want to benchmark some particular version of NativeAOT (or from a differe ```cs var config = DefaultConfig.Instance - .With(Job.ShortRun - .With(NativeAotToolchain.CreateBuilder() + .AddJob(Job.ShortRun + .WithToolchain(NativeAotToolchain.CreateBuilder() .UseNuGet( microsoftDotNetILCompilerVersion: "7.0.0-*", // the version goes here nuGetFeedUrl: "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json") // this address might change over time @@ -338,8 +337,8 @@ or explicitly in the code: ```cs var config = DefaultConfig.Instance - .With(Job.ShortRun - .With(NativeAotToolchain.CreateBuilder() + .AddJob(Job.ShortRun + .WithToolchain(NativeAotToolchain.CreateBuilder() .UseLocalBuild(@"C:\Projects\runtime\artifacts\packages\Release\Shipping\") .DisplayName("NativeAOT local build") .TargetFrameworkMoniker("net7.0") diff --git a/docs/articles/features/event-pipe-profiler.md b/docs/articles/features/event-pipe-profiler.md index 3b0ed69bc6..25d1e90386 100644 --- a/docs/articles/features/event-pipe-profiler.md +++ b/docs/articles/features/event-pipe-profiler.md @@ -7,7 +7,7 @@ name: EventPipeProfiler `EventPipeProfiler` is a cross-platform profiler that allows profile .NET code on every platform - Windows, Linux, macOS. Collected data are exported to trace files (`.speedscope.json` and `.nettrace`) which can be analyzed using [SpeedScope](https://www.speedscope.app/), [PerfView](https://github.com/Microsoft/perfview), and [Visual Studio Profiler](https://learn.microsoft.com/visualstudio/profiling/profiling-feature-tour). This new profiler is available from the 0.12.1 version. -![](https://wojciechnagorski.com/images/EventPipeProfiler/SpeedScopeAdvance.png) +![](https://wojciechnagorski.github.io/images/EventPipeProfiler/SpeedScopeAdvance.png) # Configuration diff --git a/docs/articles/features/toc.yml b/docs/articles/features/toc.yml index fe55761610..61e3a2cd54 100644 --- a/docs/articles/features/toc.yml +++ b/docs/articles/features/toc.yml @@ -11,4 +11,8 @@ - name: EtwProfiler href: etwprofiler.md - name: EventPipeProfiler - href: event-pipe-profiler.md \ No newline at end of file + href: event-pipe-profiler.md +- name: VSProfiler + href: vsprofiler.md +- name: VSTest + href: vstest.md \ No newline at end of file diff --git a/docs/articles/features/vsprofiler.md b/docs/articles/features/vsprofiler.md new file mode 100644 index 0000000000..9e8f52712d --- /dev/null +++ b/docs/articles/features/vsprofiler.md @@ -0,0 +1,71 @@ +--- +uid: docs.vsprofiler +name: VS Profiler +--- + +# Running with Visual Studio profiler +Visual Studio supports [profiler integration with BenchmarkDotNet](https://learn.microsoft.com/visualstudio/profiling/profiling-with-benchmark-dotnet) on Windows through its [Microsoft.VisualStudio.BenchmarkDotNetDiagnosers](https://www.nuget.org/packages/Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers) NuGet package. Once installed, Visual Studio specific diagnosers will capture performance data in runs and automatically open traces if launched through Visual Studio + +![](../../images/vs-profiler-demo.png) + +## How it works + +First, install the [Microsoft.VisualStudio.BenchmarkDotNetDiagnosers](https://www.nuget.org/packages/Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers) NuGet package in your benchmarking project. Next add one or more of the Visual Studio diagnosers to your benchmark to capture the relevant profiling information while benchmarking. Lastly, run your benchmarks and a diagsession will be generated. If run from Visual Studio the diagsession will automatically be opened. + +## Available Diagnosers + +* `[CPUUsageDiagnoser]` - Enables the [CPU Usage tool](https://learn.microsoft.com/visualstudio/profiling/cpu-usage). +* `[DatabaseDiagnoser]` - Enables the [Database tool](https://learn.microsoft.com/visualstudio/profiling/analyze-database) +* `[DotNetCountersDiagnoser]` - Enables the [.NET Counters tool](https://learn.microsoft.com/visualstudio/profiling/dotnet-counters-tool) +* `[DotNetObjectAllocDiagnoser]` - Enables the [.NET Object Allocation tool](https://learn.microsoft.com/visualstudio/profiling/dotnet-alloc-tool). When using this tool, you must also specify `[DotNetObjectAllocJobConfiguration]` on the benchmark. If this is missing the run will fail and you will receive an error indicating you need to add it. +* `[EventsDiagnoser]` - Enables the [Events tool](https://learn.microsoft.com/visualstudio/profiling/events-viewer) +* `[FileIODiagnoser]` - Enables the [File IO tool](https://learn.microsoft.com/visualstudio/profiling/use-file-io) + +## How to use it? + +After installing the [Microsoft.VisualStudio.BenchmarkDotNetDiagnosers](https://www.nuget.org/packages/Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers) NuGet package add the following code as a benchmark: + +```cs +using System; +using System.Security.Cryptography; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Running; +using Microsoft.VSDiagnostics; + +namespace MyBenchmarks +{ + [CPUUsageDiagnoser] + public class Md5VsSha256 + { + private const int N = 10000; + private readonly byte[] data; + + private readonly SHA256 sha256 = SHA256.Create(); + private readonly MD5 md5 = MD5.Create(); + + public Md5VsSha256() + { + data = new byte[N]; + new Random(42).NextBytes(data); + } + + [Benchmark] + public byte[] Sha256() => sha256.ComputeHash(data); + + [Benchmark] + public byte[] Md5() => md5.ComputeHash(data); + } + + public class Program + { + public static void Main(string[] args) + { + var summary = BenchmarkRunner.Run(typeof(Program).Assembly); + } + } +} +``` + +In this case we have added the `[CpuUsageDiagnoser]` to capture a CPU sampling trace. From here run the benchmark in Visual Studio (Ctrl+F5), and after the benchmark run the resulting diagsession will be displayed. Double clicking on one of the benchmark rows shown under the Benchmarks tab will filter the time selection to the specific benchmark allowing you to better isolate and investigate. + +![](../../images/vs-profiler-demo.png) \ No newline at end of file diff --git a/docs/articles/features/vstest.md b/docs/articles/features/vstest.md new file mode 100644 index 0000000000..9163cb43e5 --- /dev/null +++ b/docs/articles/features/vstest.md @@ -0,0 +1,139 @@ +--- +uid: docs.vstest +name: Running with VSTest +--- + +# Running with VSTest + +BenchmarkDotNet supports discovering and executing benchmarks through VSTest. +This provides an alternative user experience to running benchmarks with the CLI + and may be preferable for those who like their IDE's VSTest integrations that they may have used when running unit tests. + +Below is an example of running some benchmarks from the BenchmarkDotNet samples project in Visual Studio's Test Explorer. + +![](../../images/vs-testexplorer-demo.png) + +## About VSTest + +VSTest is one of the most popular test platforms in use in the .NET ecosystem, + with test frameworks such as MSTest, xUnit, and NUnit providing support for it. +Many IDEs, including Visual Studio and Rider, provide UIs for running tests through VSTest + which some users may find more accessible than running them through the command line. + +It may seem counterintuitive to run performance tests on a platform + that is designed for unit tests that expect a boolean outcome of "Passed" or "Failed". +However, VSTest provides good value as a protocol for discovering and executing tests. +In addition, we can still make use of this boolean output to indicate + if the benchmark had validation errors that caused it to fail to run. + +## Caveats and things to know + +* **The benchmark measurements may be affected by the VSTest host and your IDE!** + If you want to have accurate measurements, + it is recommended to run benchmarks through the CLI without other processes on the machine impacting performance. + This does not mean that the measurements are useless though, + it will still be able to provide useful measurements during development when comparing different approaches. +* **The test adapter will not display or execute benchmarks if optimizations are disabled.** + Please ensure you are compiling in Release mode or with `Optimize` set to true. + Using an `InProcess` toolchain will let you run your benchmarks with optimizations disabled + and will let you attach the debugger as well. +* **The VSTest adapter will not call your application's entry point.** + If you use the entry point to customize how your benchmarks are run, + you will need to do this through other means such as an assembly-level `IConfigSource`, + as shown [here](#setting-a-default-configuration). +* **The test adapter will generate an entry point for you automatically.** + The generated entry point will pass the command line arguments + and the current assembly into `BenchmarkSwitcher`, + so you can still use it in your CLI as well as in VSTest. + This means you can delete your entry point and only need to define your benchmarks. + If you want to use a custom entry point, you can still do so by setting `GenerateProgramFile` to `false` in your project file. + +## Getting started + +* **Step 1.** Install the NuGet packages. + You need to install two packages into your benchmark project: + * `BenchmarkDotNet.TestAdapter`: Implements the VSTest protocol for BenchmarkDotNet + * `Microsoft.NET.Test.Sdk`: Includes all the pieces needed for the VSTest host to run and load the VSTest adapter. +* **Step 2.** Make sure that the entry point is configured correctly. + As mentioned in the caveats section, `BenchmarkDotNet.TestAdapter` will generate an entry point for you automatically. + So, if you have an entry point already, + you will either need to delete it or set `GenerateProgramFile` to `false` in your project file to continue using your existing one. + Here is an example of a `.csproj` file based on the default Console Application template: + +```xml + + + + Exe + net8.0 + enable + enable + + false + + + + + + + + +``` + +* **Step 3.** Make sure that your IDE supports VSTest integration. + In Visual Studio, everything works out of the box. + In Rider/R# 2023.3, the VSTest integration should be activated: + * Go to the "Unit Testing" settings page. + * Rider: Settings -> Build, Execution, Deployment -> Unit Testing -> VSTest + * R#: Extensions -> ReSharper -> Options -> Tools -> Unit Testing -> Test Frameworks -> VSTest + * Make sure that the "Enable VSTest adapter support" checkbox is checked. + In future versions of Rider, this should be enabled by default. +* **Step 4.** Switch to the `Release` configuration. + As mentioned above, the TestAdapter is not able to discover and run benchmarks with optimizations disabled (by design). +* **Step 5.** Build the project. + In order to discover the benchmarks, the VSTest adapter needs to be able to find the assembly. + Once you build the project, you should observe the discovered benchmarks in your IDE's Unit Test Explorer. + +If you correctly performed all the steps above, you should be able to run your benchmarks in your IDE using embedded unit testing features. +If this doesn't work for you, don't hesitate to file [a new GitHub issue](https://github.com/dotnet/BenchmarkDotNet/issues/new). + +## Setting a default configuration + +Previously, it was common for the default configuration to be defined inside the entry point. +Since the entry point is not used when running benchmarks through VSTest, + the default configuration must be specified using a `Config` attribute that is set on the assembly instead. + +First, create a class that extends `ManualConfig` or `IConfig` which sets the default configuration you want: + +```csharp +class MyDefaultConfig : ManualConfig +{ + public MyDefaultConfig() + { + AddJob(Job.Dry); + AddLogger(Loggers.ConsoleLogger.Default); + AddValidator(JitOptimizationsValidator.DontFailOnError); + } +} +``` + +Then, set an assembly attribute with the following. + +```csharp +[assembly: Config(typeof(MyDefaultConfig))] +``` + +By convention, assembly attributes are usually defined inside `AssemblyInfo.cs` in a directory called `Properties`. + +## Viewing the results + +The full output from BenchmarkDotNet that you would have been used to seeing in the past will be sent to the "Tests" output of your IDE. +Use this view if you want to see the tabular view that compares multiple benchmarks with each other or + if you want to see the results for each individual iteration. + +One more place where you can view the results is in each individual test's output messages. +In Visual Studio, this can be viewed by clicking on the test in the Test Explorer after running it and looking at the Test Detail Summary. +Since this only displays statistics for a single benchmark case, + it does not show the tabulated view that compares multiple benchmark cases. +Instead, it displays a histogram and various other useful statistics. +Not all IDEs support displaying these output messages, so you may only be able to view the results using the "Tests" output. diff --git a/docs/articles/guides/console-args.md b/docs/articles/guides/console-args.md index 6fc5ddb245..765ad2e2bb 100644 --- a/docs/articles/guides/console-args.md +++ b/docs/articles/guides/console-args.md @@ -117,28 +117,29 @@ You can also filter the benchmarks by categories: The `--runtimes` or just `-r` allows you to run the benchmarks for selected Runtimes. Available options are: * Clr - BDN will either use Roslyn (if you run it as .NET app) or latest installed .NET SDK to build the benchmarks (if you run it as .NET Core app). -* Core - if you run it as .NET Core app, BDN will use the same target framework moniker, if you run it as .NET app it's going to use netcoreapp2.1. +* Core - if you run it as .NET Core app, BDN will use the same target framework moniker, if you run it as .NET app it's going to use net8.0. * Mono - it's going to use the Mono from `$Path`, you can override it with `--monoPath`. -* net46, net461, net462, net47, net471, net472 - to build and run benchmarks against specific .NET framework version. -* netcoreapp2.0, netcoreapp2.1, netcoreapp2.2, netcoreapp3.0, netcoreapp3.1, net5.0, net6.0, net7.0 - to build and run benchmarks against specific .NET Core version. -* nativeaot5.0, nativeaot6.0, nativeaot7.0 - to build and run benchmarks using NativeAOT. Can be customized with additional options: `--ilcPath`, `--ilCompilerVersion`. +* net46, net461, net462, net47, net471, net472, net48, net481 - to build and run benchmarks against specific .NET Framework version. +* netcoreapp3.1, net5.0, net6.0, net7.0, net8.0 - to build and run benchmarks against specific .NET (Core) version. +* nativeaot5.0, nativeaot6.0, nativeaot7.0, nativeaot8.0 - to build and run benchmarks using NativeAOT. Can be customized with additional options: `--ilcPackages`, `--ilCompilerVersion`. +* mono6.0, mono7.0, mono8.0 - to build and run benchmarks with .Net 6+ using MonoVM. -Example: run the benchmarks for .NET 4.7.2 and .NET Core 2.1: +Example: run the benchmarks for .NET 4.7.2 and .NET 8.0: ```log -dotnet run -c Release -- --runtimes net472 netcoreapp2.1 +dotnet run -c Release -- --runtimes net472 net8.0 ``` -Example: run the benchmarks for .NET Core 3.0 and latest .NET SDK installed on your PC: +Example: run the benchmarks for .NET Core 3.1 and latest .NET SDK installed on your PC: ```log -dotnet run -c Release -f netcoreapp3.0 -- --runtimes clr core +dotnet run -c Release -f netcoreapp3.1 -- --runtimes clr core ``` -But same command executed with `-f netcoreapp2.0` is going to run the benchmarks for .NET Core 2.0: +But same command executed with `-f net6.0` is going to run the benchmarks for .NET 6.0: ```log -dotnet run -c Release -f netcoreapp2.0 -- --runtimes clr core +dotnet run -c Release -f net6.0 -- --runtimes clr core ``` ## Number of invocations and iterations @@ -207,10 +208,10 @@ To perform a Mann–Whitney U Test and display the results in a dedicated column * `--statisticalTest`- Threshold for Mann–Whitney U Test. Examples: 5%, 10ms, 100ns, 1s -Example: run Mann–Whitney U test with relative ratio of 5% for all benchmarks for .NET Core 2.0 (base) vs .NET Core 2.1 (diff). .NET Core 2.0 will be baseline because it was first. +Example: run Mann–Whitney U test with relative ratio of 5% for all benchmarks for .NET 6.0 (base) vs .NET 8.0 (diff). .NET 6.0 will be baseline because it was first. ```log -dotnet run -c Release -- --filter * --runtimes netcoreapp2.0 netcoreapp2.1 --statisticalTest 5% +dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5% ``` ## More @@ -228,7 +229,7 @@ dotnet run -c Release -- --filter * --runtimes netcoreapp2.0 netcoreapp2.1 --sta * `--cli` path to dotnet cli (optional). * `--packages` the directory to restore packages to (optional). * `--coreRun` path(s) to CoreRun (optional). -* `--ilcPath` path to ILCompiler for NativeAOT. +* `--ilcPackages` path to ILCompiler for NativeAOT. * `--info` prints environment configuration including BenchmarkDotNet, OS, CPU and .NET version * `--stopOnFirstError` stop on first error. * `--help` display this help screen. @@ -242,6 +243,7 @@ dotnet run -c Release -- --filter * --runtimes netcoreapp2.0 netcoreapp2.1 --sta * `--platform` the Platform that should be used. If not specified, the host process platform is used (default). AnyCpu/X86/X64/Arm/Arm64/LoongArch64. * `--runOncePerIteration` run the benchmark exactly once per iteration. * `--buildTimeout` build timeout in seconds. +* `--wakeLock` prevents the system from entering sleep or turning off the display. None/System/Display. * `--wasmEngine` full path to a java script engine used to run the benchmarks, used by Wasm toolchain. * `--wasmMainJS` path to the test-main.js file used by Wasm toolchain. Mandatory when using \"--runtimes wasm\" * `--expose_wasm` arguments for the JavaScript engine used by Wasm toolchain. diff --git a/docs/articles/guides/good-practices.md b/docs/articles/guides/good-practices.md index 854ead5679..da4329de2b 100644 --- a/docs/articles/guides/good-practices.md +++ b/docs/articles/guides/good-practices.md @@ -4,8 +4,8 @@ Never use the Debug build for benchmarking. *Never*. The debug version of the target method can run 10–100 times slower. The release mode means that you should have `true` in your csproj file -or use [/optimize](https://learn.microsoft.com/dotnet/csharp/language-reference/compiler-options/) for `csc`. Also your never -should use an attached debugger (e.g. Visual Studio or WinDbg) during the benchmarking. The best way is +or use [/optimize](https://learn.microsoft.com/dotnet/csharp/language-reference/compiler-options/) for `csc`. Also, never +use an attached debugger (e.g. Visual Studio or WinDbg) during the benchmarking. The best way is build our benchmark in the Release mode and run it from the command line. ## Try different environments diff --git a/docs/articles/guides/nuget.md b/docs/articles/guides/nuget.md index 0e2de62a79..d1359c03d9 100644 --- a/docs/articles/guides/nuget.md +++ b/docs/articles/guides/nuget.md @@ -13,6 +13,7 @@ We have the following set of NuGet packages (you can install it directly from `n * `BenchmarkDotNet.Annotations`: Basic BenchmarkDotNet annotations for your benchmarks. * `BenchmarkDotNet.Diagnostics.Windows`: an additional optional package that provides a set of Windows diagnosers. * `BenchmarkDotNet.Diagnostics.dotTrace`: an additional optional package that provides DotTraceDiagnoser. +* `BenchmarkDotNet.Diagnostics.dotMemory`: an additional optional package that provides DotMemoryDiagnoser. * `BenchmarkDotNet.Templates`: Templates for BenchmarkDotNet. You might find other NuGet packages that start with `BenchmarkDotNet` name, but they are internal BDN packages that should not be installed manually. All that matters are the three packages mentioned above. diff --git a/docs/articles/license.md b/docs/articles/license.md index d20c387e81..6272f97306 100644 --- a/docs/articles/license.md +++ b/docs/articles/license.md @@ -1,6 +1,6 @@ ### The MIT License -Copyright (c) 2013–2023 .NET Foundation and contributors +Copyright (c) 2013–2025 .NET Foundation and contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/docs/articles/samples/IntroArgumentsSource.md b/docs/articles/samples/IntroArgumentsSource.md index 9a0bff5cf4..a5b3692870 100644 --- a/docs/articles/samples/IntroArgumentsSource.md +++ b/docs/articles/samples/IntroArgumentsSource.md @@ -11,7 +11,7 @@ You can mark one or several fields or properties in your class by the [`[ArgumentsSource]`](xref:BenchmarkDotNet.Attributes.ArgumentsSourceAttribute) attribute. In this attribute, you have to specify the name of public method/property which is going to provide the values (something that implements `IEnumerable`). - The source must be within benchmarked type! +The source may be instance or static. If the source is not in the same type as the benchmark, the type containing the source must be specified in the attribute constructor. ### Source code diff --git a/docs/articles/samples/IntroDotMemoryDiagnoser.md b/docs/articles/samples/IntroDotMemoryDiagnoser.md new file mode 100644 index 0000000000..e16e96a83c --- /dev/null +++ b/docs/articles/samples/IntroDotMemoryDiagnoser.md @@ -0,0 +1,22 @@ +--- +uid: BenchmarkDotNet.Samples.IntroDotMemoryDiagnoser +--- + +## Sample: IntroDotMemoryDiagnoser + +If you want to get a memory allocation profile of your benchmarks, just add the `[DotMemoryDiagnoser]` attribute, as shown below. +As a result, BenchmarkDotNet performs bonus benchmark runs using attached + [dotMemory Command-Line Profiler](https://www.jetbrains.com/help/dotmemory/Working_with_dotMemory_Command-Line_Profiler.html). +The obtained dotMemory workspaces are saved to the `artifacts` folder. +These dotMemory workspaces can be opened using the [standalone dotMemory](https://www.jetbrains.com/dotmemory/), + or [dotMemory in Rider](https://www.jetbrains.com/help/rider/Memory_profiling_of_.NET_code.html). + +### Source code + +[!code-csharp[IntroDotMemoryDiagnoser.cs](../../../samples/BenchmarkDotNet.Samples/IntroDotMemoryDiagnoser.cs)] + +### Links + +* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroDotMemoryDiagnoser + +--- \ No newline at end of file diff --git a/docs/articles/samples/IntroNuGet.md b/docs/articles/samples/IntroNuGet.md index 53d9ac54f8..1cc9af84a2 100644 --- a/docs/articles/samples/IntroNuGet.md +++ b/docs/articles/samples/IntroNuGet.md @@ -1,10 +1,10 @@ ---- +--- uid: BenchmarkDotNet.Samples.IntroNuGet --- ## Sample: IntroNuGet -You can set specific versions of NuGet dependencies for each job. +You can set specific versions of NuGet dependencies for each job using MsBuild properties in your csproj. It allows comparing different versions of the same package (if there are no breaking changes in API). ### Source code @@ -13,14 +13,11 @@ It allows comparing different versions of the same package (if there are no brea ### Output -| Method | Job | NuGetReferences | Mean | Error | StdDev | -|------------------------- |------- |----------------------- |---------:|----------:|----------:| -| SerializeAnonymousObject | 10.0.1 | Newtonsoft.Json 10.0.1 | 2.926 us | 0.0795 us | 0.0283 us | -| SerializeAnonymousObject | 10.0.2 | Newtonsoft.Json 10.0.2 | 2.877 us | 0.5928 us | 0.2114 us | -| SerializeAnonymousObject | 10.0.3 | Newtonsoft.Json 10.0.3 | 2.706 us | 0.1251 us | 0.0446 us | -| SerializeAnonymousObject | 11.0.1 | Newtonsoft.Json 11.0.1 | 2.778 us | 0.5037 us | 0.1796 us | -| SerializeAnonymousObject | 11.0.2 | Newtonsoft.Json 11.0.2 | 2.644 us | 0.0609 us | 0.0217 us | -| SerializeAnonymousObject | 9.0.1 | Newtonsoft.Json 9.0.1 | 2.722 us | 0.3552 us | 0.1267 us | +| Method | Job | Arguments | Mean | Error | StdDev | +|-------------------------- |------- |-------------------- |---------:|----------:|----------:| +| ToImmutableArrayBenchmark | v9.0.0 | /p:SciVersion=9.0.0 | 1.173 μs | 0.0057 μs | 0.0086 μs | +| ToImmutableArrayBenchmark | v9.0.3 | /p:SciVersion=9.0.3 | 1.173 μs | 0.0038 μs | 0.0058 μs | +| ToImmutableArrayBenchmark | v9.0.5 | /p:SciVersion=9.0.5 | 1.172 μs | 0.0107 μs | 0.0157 μs | ### Links diff --git a/docs/articles/samples/IntroParamsSource.md b/docs/articles/samples/IntroParamsSource.md index 5164d2514a..e631fbf1db 100644 --- a/docs/articles/samples/IntroParamsSource.md +++ b/docs/articles/samples/IntroParamsSource.md @@ -10,7 +10,7 @@ You can mark one or several fields or properties in your class by the [`[Params]`](xref:BenchmarkDotNet.Attributes.ParamsAttribute) attribute. In this attribute, you have to specify the name of public method/property which is going to provide the values (something that implements `IEnumerable`). -The source must be within benchmarked type! +The source may be instance or static. If the source is not in the same type as the benchmark, the type containing the source must be specified in the attribute constructor. ### Source code diff --git a/docs/articles/samples/IntroSummaryStyle.md b/docs/articles/samples/IntroSummaryStyle.md new file mode 100644 index 0000000000..470bb901a1 --- /dev/null +++ b/docs/articles/samples/IntroSummaryStyle.md @@ -0,0 +1,51 @@ +--- +uid: BenchmarkDotNet.SummaryStyle +--- + +## SummaryStyle in BenchmarkDotNet + +`SummaryStyle` is a class in BenchmarkDotNet that allows customization of the summary reports of benchmark results. It offers several properties to fine-tune how the results are displayed. + +### Usage + +You can customize the summary report by specifying various properties of `SummaryStyle`. These properties include formatting options like whether to print units in the header or content, setting the maximum width for parameter columns, and choosing units for size and time measurements. + +### Source Code + +[!code-csharp[IntroSummaryStyle.cs](../../../samples/BenchmarkDotNet.Samples/IntroSummaryStyle.cs)] + +### Properties + +- `PrintUnitsInHeader`: Boolean to indicate if units should be printed in the header. +- `PrintUnitsInContent`: Boolean to control unit printing in the content. +- `PrintZeroValuesInContent`: Determines if zero values should be printed. +- `MaxParameterColumnWidth`: Integer defining the max width for parameter columns. +- `SizeUnit`: Optional `SizeUnit` to specify the unit for size measurements. +- `TimeUnit`: Optional `TimeUnit` for time measurement units. +- `CultureInfo`: `CultureInfo` to define culture-specific formatting. + +### Example Output + +Using SummaryStyle options: + +```markdown +| Method | N | Mean [ns] | Error [ns] | StdDev [ns] | +|------- |---- |--------------:|-----------:|------------:| +| Sleep | 10 | 15,644,973.1 | 32,808.7 | 30,689.3 | +| Sleep | 100 | 109,440,686.7 | 236,673.8 | 221,384.8 | +``` + +Default: + +```markdown +| Method | N | Mean | Error | StdDev | +|------- |---- |----------:|---------:|---------:| +| Sleep | 10 | 15.65 ms | 0.039 ms | 0.034 ms | +| Sleep | 100 | 109.20 ms | 0.442 ms | 0.392 ms | +``` + +### Links + +* @docs.SummaryStyle +* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroSummaryStyle + diff --git a/docs/articles/samples/IntroVisualStudioProfiler.md b/docs/articles/samples/IntroVisualStudioProfiler.md new file mode 100644 index 0000000000..8d35498f21 --- /dev/null +++ b/docs/articles/samples/IntroVisualStudioProfiler.md @@ -0,0 +1,23 @@ +--- +uid: BenchmarkDotNet.Samples.IntroVisualStudioProfiler +--- + +## Sample: Visual Studio Profiler + +Using the [Microsoft.VisualStudio.BenchmarkDotNetDiagnosers](https://www.nuget.org/packages/Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers) NuGet package you can capture performance profiles of your benchmarks that can be opened in Visual Studio. + +### Source code + +[!code-csharp[IntroVisualStudioDiagnoser.cs](../../../samples/BenchmarkDotNet.Samples/IntroVisualStudioDiagnoser.cs)] + +### Output +The output will contain a path to the collected diagsession and automatically open in Visual Studio when launched from it. + +```markdown +// * Diagnostic Output - VSDiagnosticsDiagnoser * +Collection result moved to 'C:\Work\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\net8.0\BenchmarkDotNet.Artifacts\BenchmarkDotNet_IntroVisualStudioProfiler_20241205_192056.diagsession'. +Session : {d54ebddb-2d6d-404f-b1da-10acbc89635f} + Stopped +Exported diagsession file: C:\Work\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\net8.0\BenchmarkDotNet.Artifacts\BenchmarkDotNet_IntroVisualStudioProfiler_20241205_192056.diagsession. +Opening diagsession in VisualStudio: 15296 +``` \ No newline at end of file diff --git a/docs/articles/samples/IntroWakeLock.md b/docs/articles/samples/IntroWakeLock.md new file mode 100644 index 0000000000..333b3e4345 --- /dev/null +++ b/docs/articles/samples/IntroWakeLock.md @@ -0,0 +1,32 @@ +--- +uid: BenchmarkDotNet.Samples.IntroWakeLock +--- + +## Sample: IntroWakeLock + +Running benchmarks may sometimes take enough time such that the system enters sleep or turns off the display. + +Using a WakeLock prevents the Windows system doing so. + +### Source code + +[!code-csharp[IntroWakeLock.cs](../../../samples/BenchmarkDotNet.Samples/IntroWakeLock.cs)] + +### Command line + +``` +--wakeLock None +``` +``` +--wakeLock System +``` +``` +--wakeLock Display +``` + +### Links + +* @BenchmarkDotNet.Attributes.WakeLockAttribute +* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroWakeLock + +--- diff --git a/docs/articles/samples/toc.yml b/docs/articles/samples/toc.yml index 8098a637c3..8dab3b5c6b 100644 --- a/docs/articles/samples/toc.yml +++ b/docs/articles/samples/toc.yml @@ -38,6 +38,8 @@ href: IntroDisassemblyRyuJit.md - name: IntroDotTraceDiagnoser href: IntroDotTraceDiagnoser.md +- name: IntroDotMemoryDiagnoser + href: IntroDotMemoryDiagnoser.md - name: IntroEnvVars href: IntroEnvVars.md - name: IntroEventPipeProfiler @@ -122,6 +124,10 @@ href: IntroTagColumn.md - name: IntroTailcall href: IntroTailcall.md +- name: IntroVisualStudioProfiler + href: IntroVisualStudioProfiler.md +- name: IntroWakeLock + href: IntroWakeLock.md - name: IntroWasm href: IntroWasm.md - name: IntroUnicode diff --git a/docs/articles/toc.yml b/docs/articles/toc.yml index 29ad2cce16..60f1d5e11a 100644 --- a/docs/articles/toc.yml +++ b/docs/articles/toc.yml @@ -14,5 +14,5 @@ href: faq.md - name: Team href: team.md -- name: Licence - href: license.md \ No newline at end of file +- name: License + href: license.md diff --git a/docs/changelog/.gitignore b/docs/changelog/.gitignore deleted file mode 100644 index 9f626be1d8..0000000000 --- a/docs/changelog/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.md -*.yml \ No newline at end of file diff --git a/docs/docfx.json b/docs/docfx.json index 52ed261bb6..291ff58dbd 100644 --- a/docs/docfx.json +++ b/docs/docfx.json @@ -64,7 +64,7 @@ "globalMetadata" : { "_appTitle" : "BenchmarkDotNet", - "_appFooter" : "Copyright © 2013–2023 .NET Foundation and contributors", + "_appFooter" : "Copyright © 2013–2024 .NET Foundation and contributors", "_appLogoPath" : "logo/icon.svg", "_appFaviconPath": "logo/icon-32.png", "_enableSearch": true, diff --git a/docs/images/vs-profiler-demo.png b/docs/images/vs-profiler-demo.png new file mode 100644 index 0000000000..928f012df1 Binary files /dev/null and b/docs/images/vs-profiler-demo.png differ diff --git a/docs/images/vs-testexplorer-demo.png b/docs/images/vs-testexplorer-demo.png new file mode 100644 index 0000000000..91232e30d6 Binary files /dev/null and b/docs/images/vs-testexplorer-demo.png differ diff --git a/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj b/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj index 55045658d8..6359340234 100644 --- a/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj +++ b/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj @@ -5,13 +5,17 @@ Exe - net462;net7.0 + net462;net8.0 + false + + false + @@ -19,11 +23,8 @@ - - all - runtime; build; native; contentfiles; analyzers - - - + + + diff --git a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj index 180148454a..948849aa05 100644 --- a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj +++ b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj @@ -2,7 +2,7 @@ BenchmarkDotNet.Samples - net7.0;net462 + net8.0;net462 true BenchmarkDotNet.Samples Exe @@ -11,18 +11,33 @@ AnyCPU true $(NoWarn);CA1018;CA5351;CA1825 + + false + + false - + + + + 9.0.0 + + + - - + + + + + + + diff --git a/samples/BenchmarkDotNet.Samples/IntroArgumentsSource.cs b/samples/BenchmarkDotNet.Samples/IntroArgumentsSource.cs index 33a8d90c0c..493422e51f 100644 --- a/samples/BenchmarkDotNet.Samples/IntroArgumentsSource.cs +++ b/samples/BenchmarkDotNet.Samples/IntroArgumentsSource.cs @@ -20,10 +20,13 @@ public class IntroArgumentsSource } [Benchmark] - [ArgumentsSource(nameof(TimeSpans))] + [ArgumentsSource(typeof(BenchmarkArguments), nameof(BenchmarkArguments.TimeSpans))] // when the arguments come from a different type, specify that type here public void SingleArgument(TimeSpan time) => Thread.Sleep(time); + } - public IEnumerable TimeSpans() // for single argument it's an IEnumerable of objects (object) + public static class BenchmarkArguments + { + public static IEnumerable TimeSpans() // for single argument it's an IEnumerable of objects (object) { yield return TimeSpan.FromMilliseconds(10); yield return TimeSpan.FromMilliseconds(100); diff --git a/samples/BenchmarkDotNet.Samples/IntroDotMemoryDiagnoser.cs b/samples/BenchmarkDotNet.Samples/IntroDotMemoryDiagnoser.cs new file mode 100644 index 0000000000..894bfc6c34 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroDotMemoryDiagnoser.cs @@ -0,0 +1,46 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Diagnostics.dotMemory; +using System.Collections.Generic; + +namespace BenchmarkDotNet.Samples +{ + // Profile benchmarks via dotMemory SelfApi profiling for all jobs + [DotMemoryDiagnoser] + [SimpleJob] // external-process execution + [InProcess] // in-process execution + public class IntroDotMemoryDiagnoser + { + [Params(1024)] + public int Size; + + private byte[] dataArray; + private IEnumerable dataEnumerable; + + [GlobalSetup] + public void Setup() + { + dataArray = new byte[Size]; + dataEnumerable = dataArray; + } + + [Benchmark] + public int IterateArray() + { + var count = 0; + foreach (var _ in dataArray) + count++; + + return count; + } + + [Benchmark] + public int IterateEnumerable() + { + var count = 0; + foreach (var _ in dataEnumerable) + count++; + + return count; + } + } +} \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroDotTraceDiagnoser.cs b/samples/BenchmarkDotNet.Samples/IntroDotTraceDiagnoser.cs index 351207c78b..047e6ee059 100644 --- a/samples/BenchmarkDotNet.Samples/IntroDotTraceDiagnoser.cs +++ b/samples/BenchmarkDotNet.Samples/IntroDotTraceDiagnoser.cs @@ -3,16 +3,11 @@ namespace BenchmarkDotNet.Samples { - // Enables dotTrace profiling for all jobs + // Profile benchmarks via dotTrace SelfApi profiling for all jobs + // See: https://www.nuget.org/packages/JetBrains.Profiler.SelfApi [DotTraceDiagnoser] - // Adds the default "external-process" job - // Profiling is performed using dotTrace command-line Tools - // See: https://www.jetbrains.com/help/profiler/Performance_Profiling__Profiling_Using_the_Command_Line.html - [SimpleJob] - // Adds an "in-process" job - // Profiling is performed using dotTrace SelfApi - // NuGet reference: https://www.nuget.org/packages/JetBrains.Profiler.SelfApi - [InProcess] + [SimpleJob] // external-process execution + [InProcess] // in-process execution public class IntroDotTraceDiagnoser { [Benchmark] diff --git a/samples/BenchmarkDotNet.Samples/IntroEnvVars.cs b/samples/BenchmarkDotNet.Samples/IntroEnvVars.cs index 59bd2db82f..66f5197119 100644 --- a/samples/BenchmarkDotNet.Samples/IntroEnvVars.cs +++ b/samples/BenchmarkDotNet.Samples/IntroEnvVars.cs @@ -10,13 +10,14 @@ public class IntroEnvVars { private class ConfigWithCustomEnvVars : ManualConfig { - private const string JitNoInline = "COMPlus_JitNoInline"; - public ConfigWithCustomEnvVars() { - AddJob(Job.Default.WithRuntime(CoreRuntime.Core21).WithId("Inlining enabled")); - AddJob(Job.Default.WithRuntime(CoreRuntime.Core21) - .WithEnvironmentVariables(new EnvironmentVariable(JitNoInline, "1")) + AddJob(Job.Default.WithRuntime(CoreRuntime.Core80).WithId("Inlining enabled")); + AddJob(Job.Default.WithRuntime(CoreRuntime.Core80) + .WithEnvironmentVariables([ + new EnvironmentVariable("DOTNET_JitNoInline", "1"), + new EnvironmentVariable("COMPlus_JitNoInline", "1") + ]) .WithId("Inlining disabled")); } } @@ -27,4 +28,4 @@ public void Foo() // Benchmark body } } -} \ No newline at end of file +} diff --git a/samples/BenchmarkDotNet.Samples/IntroFluentConfigBuilder.cs b/samples/BenchmarkDotNet.Samples/IntroFluentConfigBuilder.cs index 94c34c1cbd..4ac29919ee 100644 --- a/samples/BenchmarkDotNet.Samples/IntroFluentConfigBuilder.cs +++ b/samples/BenchmarkDotNet.Samples/IntroFluentConfigBuilder.cs @@ -38,7 +38,7 @@ public static void Run() .Run( DefaultConfig.Instance .AddJob(Job.Default.WithRuntime(ClrRuntime.Net462)) - .AddJob(Job.Default.WithRuntime(CoreRuntime.Core21)) + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80)) .AddValidator(ExecutionValidator.FailOnError)); } } diff --git a/samples/BenchmarkDotNet.Samples/IntroNativeMemory.cs b/samples/BenchmarkDotNet.Samples/IntroNativeMemory.cs index f968696b43..75f636e87e 100644 --- a/samples/BenchmarkDotNet.Samples/IntroNativeMemory.cs +++ b/samples/BenchmarkDotNet.Samples/IntroNativeMemory.cs @@ -3,6 +3,7 @@ using System.Runtime.InteropServices; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Diagnostics.Windows.Configs; +using BenchmarkDotNet.Filters; namespace BenchmarkDotNet.Samples { @@ -11,7 +12,8 @@ namespace BenchmarkDotNet.Samples [MemoryDiagnoser] public class IntroNativeMemory { - [Benchmark] +#pragma warning disable CA1416 + [Benchmark, WindowsOnly] public void BitmapWithLeaks() { var flag = new Bitmap(200, 100); @@ -20,7 +22,7 @@ public void BitmapWithLeaks() graphics.DrawLine(blackPen, 100, 100, 500, 100); } - [Benchmark] + [Benchmark, WindowsOnly] public void Bitmap() { using (var flag = new Bitmap(200, 100)) @@ -34,6 +36,7 @@ public void Bitmap() } } } +#pragma warning restore CA1416 private const int Size = 20; // Greater value could cause System.OutOfMemoryException for test with memory leaks. private int ArraySize = Size * Marshal.SizeOf(typeof(int)); @@ -52,5 +55,13 @@ public unsafe void AllocHGlobalWithLeaks() IntPtr unmanagedHandle = Marshal.AllocHGlobal(ArraySize); Span unmanaged = new Span(unmanagedHandle.ToPointer(), ArraySize); } + + private class WindowsOnlyAttribute : FilterConfigBaseAttribute + { + public WindowsOnlyAttribute() + : base(new SimpleFilter(_ => RuntimeInformation.IsOSPlatform(OSPlatform.Windows))) + { + } + } } } diff --git a/samples/BenchmarkDotNet.Samples/IntroNuGet.cs b/samples/BenchmarkDotNet.Samples/IntroNuGet.cs index 2eb31cd89a..850a9ceebf 100644 --- a/samples/BenchmarkDotNet.Samples/IntroNuGet.cs +++ b/samples/BenchmarkDotNet.Samples/IntroNuGet.cs @@ -1,8 +1,9 @@ using System; +using System.Collections.Immutable; +using System.Linq; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Jobs; -using Newtonsoft.Json; namespace BenchmarkDotNet.Samples { @@ -10,33 +11,49 @@ namespace BenchmarkDotNet.Samples /// Benchmarks between various versions of a NuGet package /// /// - /// Only supported with the CsProjCoreToolchain toolchain + /// Only supported with CsProj toolchains. /// [Config(typeof(Config))] public class IntroNuGet { - // Specify jobs with different versions of the same NuGet package to benchmark. - // The NuGet versions referenced on these jobs must be greater or equal to the - // same NuGet version referenced in this benchmark project. - // Example: This benchmark project references Newtonsoft.Json 9.0.1 + // Setup your csproj like this: + /* + + + 9.0.0 + + + + + */ + // All versions of the package must be source-compatible with your benchmark code. private class Config : ManualConfig { public Config() { - var baseJob = Job.MediumRun; + string[] targetVersions = [ + "9.0.0", + "9.0.3", + "9.0.5", + ]; - AddJob(baseJob.WithNuGet("Newtonsoft.Json", "11.0.2").WithId("11.0.2")); - AddJob(baseJob.WithNuGet("Newtonsoft.Json", "11.0.1").WithId("11.0.1")); - AddJob(baseJob.WithNuGet("Newtonsoft.Json", "10.0.3").WithId("10.0.3")); - AddJob(baseJob.WithNuGet("Newtonsoft.Json", "10.0.2").WithId("10.0.2")); - AddJob(baseJob.WithNuGet("Newtonsoft.Json", "10.0.1").WithId("10.0.1")); - AddJob(baseJob.WithNuGet("Newtonsoft.Json", "9.0.1").WithId("9.0.1")); + foreach (var version in targetVersions) + { + AddJob(Job.MediumRun + .WithMsBuildArguments($"/p:SciVersion={version}") + .WithId($"v{version}") + ); + } } } + private static readonly Random rand = new Random(Seed: 0); + private static readonly double[] values = Enumerable.Range(1, 10_000).Select(x => rand.NextDouble()).ToArray(); + [Benchmark] - public void SerializeAnonymousObject() - => JsonConvert.SerializeObject( - new { hello = "world", price = 1.99, now = DateTime.UtcNow }); + public void ToImmutableArrayBenchmark() + { + var results = values.ToImmutableArray(); + } } } \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroParamsSource.cs b/samples/BenchmarkDotNet.Samples/IntroParamsSource.cs index cfd00d50f4..8257439ddf 100644 --- a/samples/BenchmarkDotNet.Samples/IntroParamsSource.cs +++ b/samples/BenchmarkDotNet.Samples/IntroParamsSource.cs @@ -20,7 +20,16 @@ public class IntroParamsSource // public static method public static IEnumerable ValuesForB() => new[] { 10, 20 }; + // public field getting its params from a method in another type + [ParamsSource(typeof(ParamsValues), nameof(ParamsValues.ValuesForC))] + public int C; + [Benchmark] - public void Benchmark() => Thread.Sleep(A + B + 5); + public void Benchmark() => Thread.Sleep(A + B + C + 5); + } + + public static class ParamsValues + { + public static IEnumerable ValuesForC() => new[] { 1000, 2000 }; } } \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroSmokeEmptyBasic.cs b/samples/BenchmarkDotNet.Samples/IntroSmokeEmptyBasic.cs new file mode 100644 index 0000000000..39783dc74a --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroSmokeEmptyBasic.cs @@ -0,0 +1,82 @@ +using BenchmarkDotNet.Attributes; + +namespace BenchmarkDotNet.Samples; + +[DisassemblyDiagnoser] +public class IntroSmokeEmptyBasic +{ + [Benchmark] public void Void1() {} + [Benchmark] public void Void2() {} + [Benchmark] public void Void3() {} + [Benchmark] public void Void4() {} + + [Benchmark] public byte Byte1() => 0; + [Benchmark] public byte Byte2() => 0; + [Benchmark] public byte Byte3() => 0; + [Benchmark] public byte Byte4() => 0; + + [Benchmark] public sbyte Sbyte1() => 0; + [Benchmark] public sbyte Sbyte2() => 0; + [Benchmark] public sbyte Sbyte3() => 0; + [Benchmark] public sbyte Sbyte4() => 0; + + [Benchmark] public short Short1() => 0; + [Benchmark] public short Short2() => 0; + [Benchmark] public short Short3() => 0; + [Benchmark] public short Short4() => 0; + + [Benchmark] public ushort Ushort1() => 0; + [Benchmark] public ushort Ushort2() => 0; + [Benchmark] public ushort Ushort3() => 0; + [Benchmark] public ushort Ushort4() => 0; + + [Benchmark] public int Int1() => 0; + [Benchmark] public int Int2() => 0; + [Benchmark] public int Int3() => 0; + [Benchmark] public int Int4() => 0; + + [Benchmark] public uint Uint1() => 0u; + [Benchmark] public uint Uint2() => 0u; + [Benchmark] public uint Uint3() => 0u; + [Benchmark] public uint Uint4() => 0u; + + [Benchmark] public bool Bool1() => false; + [Benchmark] public bool Bool2() => false; + [Benchmark] public bool Bool3() => false; + [Benchmark] public bool Bool4() => false; + + [Benchmark] public char Char1() => 'a'; + [Benchmark] public char Char2() => 'a'; + [Benchmark] public char Char3() => 'a'; + [Benchmark] public char Char4() => 'a'; + + [Benchmark] public float Float1() => 0f; + [Benchmark] public float Float2() => 0f; + [Benchmark] public float Float3() => 0f; + [Benchmark] public float Float4() => 0f; + + [Benchmark] public double Double1() => 0d; + [Benchmark] public double Double2() => 0d; + [Benchmark] public double Double3() => 0d; + [Benchmark] public double Double4() => 0d; + + [Benchmark] public long Long1() => 0L; + [Benchmark] public long Long2() => 0L; + [Benchmark] public long Long3() => 0L; + [Benchmark] public long Long4() => 0L; + + [Benchmark] public ulong Ulong1() => 0uL; + [Benchmark] public ulong Ulong2() => 0uL; + [Benchmark] public ulong Ulong3() => 0uL; + [Benchmark] public ulong Ulong4() => 0uL; + + [Benchmark] public string String1() => ""; + [Benchmark] public string String2() => ""; + [Benchmark] public string String3() => ""; + [Benchmark] public string String4() => ""; + + [Benchmark] public object? Object1() => null; + [Benchmark] public object? Object2() => null; + [Benchmark] public object? Object3() => null; + [Benchmark] public object? Object4() => null; +} \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroSmokeIncrements.cs b/samples/BenchmarkDotNet.Samples/IntroSmokeIncrements.cs new file mode 100644 index 0000000000..6dfd15433d --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroSmokeIncrements.cs @@ -0,0 +1,138 @@ +using BenchmarkDotNet.Attributes; + +namespace BenchmarkDotNet.Samples; + +public class IntroSmokeIncrements +{ + public int Field; + + [Benchmark] + public void Increment01() + { + Field++; + } + + [Benchmark] + public void Increment02() + { + Field++; + Field++; + } + + [Benchmark] + public void Increment03() + { + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment04() + { + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment05() + { + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment06() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment07() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment08() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment09() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment10() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment20() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } +} \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroSmokeValueTypes.cs b/samples/BenchmarkDotNet.Samples/IntroSmokeValueTypes.cs new file mode 100644 index 0000000000..66bce4d921 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroSmokeValueTypes.cs @@ -0,0 +1,70 @@ +using System; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Environments; + +namespace BenchmarkDotNet.Samples; + +[MemoryDiagnoser, DisassemblyDiagnoser] +public class IntroSmokeValueTypes +{ + [Benchmark] public Jit ReturnEnum() => Jit.RyuJit; + + [Benchmark] public DateTime ReturnDateTime() => new DateTime(); + + [Benchmark] public DateTime? ReturnNullableDateTime() => new DateTime(); + [Benchmark] public int? ReturnNullableInt() => 0; + + public struct StructWithReferencesOnly { public object _ref; } + [Benchmark] public StructWithReferencesOnly ReturnStructWithReferencesOnly() => new StructWithReferencesOnly(); + + public struct EmptyStruct { } + [Benchmark] public EmptyStruct ReturnEmptyStruct() => new EmptyStruct(); + + [Benchmark] public ValueTuple ReturnGenericStructOfValueType() => new ValueTuple(0); + [Benchmark] public ValueTuple ReturnGenericStructOfReferenceType() => new ValueTuple(null); + + [Benchmark] public ValueTask ReturnValueTaskOfValueType() => new ValueTask(0); + [Benchmark] public ValueTask ReturnValueTaskOfReferenceType() => new ValueTask(result: null); + + [Benchmark] public byte ReturnByte() => 0; + public struct Byte1 { public byte _1; } + [Benchmark] public Byte1 ReturnByte1() => new Byte1(); + public struct Byte2 { public byte _1, _2; } + [Benchmark] public Byte2 ReturnByte2() => new Byte2(); + public struct Byte3 { public byte _1, _2, _3; } + [Benchmark] public Byte3 ReturnByte3() => new Byte3(); + public struct Byte4 { public byte _1, _2, _3, _4; } + [Benchmark] public Byte4 ReturnByte4() => new Byte4(); + + [Benchmark] public short ReturnShort() => 0; + public struct Short1 { public short _1; } + [Benchmark] public Short1 ReturnShort1() => new Short1(); + public struct Short2 { public short _1, _2; } + [Benchmark] public Short2 ReturnShort2() => new Short2(); + public struct Short3 { public short _1, _2, _3; } + [Benchmark] public Short3 ReturnShort3() => new Short3(); + public struct Short4 { public short _1, _2, _3, _4; } + [Benchmark] public Short4 ReturnShort4() => new Short4(); + + [Benchmark] public int ReturnInt() => 0; + public struct Int1 { public int _1; } + [Benchmark] public Int1 ReturnInt1() => new Int1(); + public struct Int2 { public int _1, _2; } + [Benchmark] public Int2 ReturnInt2() => new Int2(); + public struct Int3 { public int _1, _2, _3; } + [Benchmark] public Int3 ReturnInt3() => new Int3(); + public struct Int4 { public int _1, _2, _3, _4; } + [Benchmark] public Int4 ReturnInt4() => new Int4(); + + [Benchmark] public long ReturnLong() => 0; + public struct Long1 { public long _1; } + [Benchmark] public Long1 ReturnLong1() => new Long1(); + public struct Long2 { public long _1, _2; } + [Benchmark] public Long2 ReturnLong2() => new Long2(); + public struct Long3 { public long _1, _2, _3; } + [Benchmark] public Long3 ReturnLong3() => new Long3(); + public struct Long4 { public long _1, _2, _3, _4; } + [Benchmark] public Long4 ReturnLong4() => new Long4(); +} +// ReSharper restore InconsistentNaming \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs b/samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs index bbdf85aad5..6351281aa6 100644 --- a/samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs +++ b/samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs @@ -1,14 +1,10 @@ using System.Threading; using BenchmarkDotNet.Attributes; -using Perfolizer.Mathematics.SignificanceTesting; -using Perfolizer.Mathematics.Thresholds; namespace BenchmarkDotNet.Samples { - [StatisticalTestColumn(StatisticalTestKind.Welch, ThresholdUnit.Microseconds, 1, true)] - [StatisticalTestColumn(StatisticalTestKind.MannWhitney, ThresholdUnit.Microseconds, 1, true)] - [StatisticalTestColumn(StatisticalTestKind.Welch, ThresholdUnit.Ratio, 0.03, true)] - [StatisticalTestColumn(StatisticalTestKind.MannWhitney, ThresholdUnit.Ratio, 0.03, true)] + [StatisticalTestColumn("500us")] + [StatisticalTestColumn("3%")] [SimpleJob(warmupCount: 0, iterationCount: 5)] public class IntroStatisticalTesting { diff --git a/samples/BenchmarkDotNet.Samples/IntroSummaryStyle.cs b/samples/BenchmarkDotNet.Samples/IntroSummaryStyle.cs new file mode 100644 index 0000000000..6a9830be18 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroSummaryStyle.cs @@ -0,0 +1,39 @@ +using System.Globalization; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Reports; +using Perfolizer.Horology; +using Perfolizer.Metrology; + +namespace BenchmarkDotNet.Samples +{ + [Config(typeof(Config))] + public class IntroSummaryStyle + { + private class Config : ManualConfig + { + public Config() + { + // Configure the summary style here + var summaryStyle = new SummaryStyle + ( + cultureInfo: CultureInfo.InvariantCulture, + printUnitsInHeader: true, + printUnitsInContent: false, + sizeUnit: SizeUnit.KB, + timeUnit: TimeUnit.Nanosecond, + maxParameterColumnWidth: 20 + + ); + + WithSummaryStyle(summaryStyle); + } + } + + [Params(10, 100)] + public int N; + + [Benchmark] + public void Sleep() => System.Threading.Thread.Sleep(N); + } +} \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroVisualStudioDiagnoser.cs b/samples/BenchmarkDotNet.Samples/IntroVisualStudioDiagnoser.cs new file mode 100644 index 0000000000..e513f39320 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroVisualStudioDiagnoser.cs @@ -0,0 +1,23 @@ +using System; +using BenchmarkDotNet.Attributes; +using Microsoft.VSDiagnostics; + +namespace BenchmarkDotNet.Samples +{ + // Enables profiling with the CPU Usage tool + // See: https://learn.microsoft.com/visualstudio/profiling/profiling-with-benchmark-dotnet + [CPUUsageDiagnoser] + public class IntroVisualStudioProfiler + { + private readonly Random rand = new Random(42); + + [Benchmark] + public void BurnCPU() + { + for (int i = 0; i < 100000; ++i) + { + rand.Next(1, 100); + } + } + } +} \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroWakeLock.cs b/samples/BenchmarkDotNet.Samples/IntroWakeLock.cs new file mode 100644 index 0000000000..099b347c7b --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroWakeLock.cs @@ -0,0 +1,30 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using System; +using System.Threading; + +// *** Attribute Style applied to Assembly *** +[assembly: WakeLock(WakeLockType.System)] + +namespace BenchmarkDotNet.Samples; + +// *** Attribute Style *** +[WakeLock(WakeLockType.Display)] +public class IntroWakeLock +{ + [Benchmark] + public void LongRunning() => Thread.Sleep(TimeSpan.FromSeconds(10)); +} + +// *** Object Style *** +[Config(typeof(Config))] +public class IntroWakeLockObjectStyle +{ + private class Config : ManualConfig + { + public Config() => WakeLock = WakeLockType.System; + } + + [Benchmark] + public void LongRunning() => Thread.Sleep(TimeSpan.FromSeconds(10)); +} diff --git a/src/BenchmarkDotNet.Annotations/Attributes/ArgumentsSourceAttribute.cs b/src/BenchmarkDotNet.Annotations/Attributes/ArgumentsSourceAttribute.cs index a7d3fe38bb..f4836e329a 100644 --- a/src/BenchmarkDotNet.Annotations/Attributes/ArgumentsSourceAttribute.cs +++ b/src/BenchmarkDotNet.Annotations/Attributes/ArgumentsSourceAttribute.cs @@ -6,7 +6,18 @@ namespace BenchmarkDotNet.Attributes public class ArgumentsSourceAttribute : PriorityAttribute { public string Name { get; } + public Type? Type { get; } - public ArgumentsSourceAttribute(string name) => Name = name; + public ArgumentsSourceAttribute(string name) + { + Name = name; + Type = null; + } + + public ArgumentsSourceAttribute(Type type, string name) + { + Name = name; + Type = type; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet.Annotations/Attributes/ParamsAttribute.cs b/src/BenchmarkDotNet.Annotations/Attributes/ParamsAttribute.cs index 22148ffa64..aa47ebdf96 100644 --- a/src/BenchmarkDotNet.Annotations/Attributes/ParamsAttribute.cs +++ b/src/BenchmarkDotNet.Annotations/Attributes/ParamsAttribute.cs @@ -5,7 +5,7 @@ namespace BenchmarkDotNet.Attributes [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property)] public class ParamsAttribute : PriorityAttribute { - public object?[] Values { get; } + public object?[] Values { get; protected set; } // CLS-Compliant Code requires a constructor without an array in the argument list public ParamsAttribute() => Values = new object[0]; diff --git a/src/BenchmarkDotNet.Annotations/Attributes/ParamsSourceAttribute.cs b/src/BenchmarkDotNet.Annotations/Attributes/ParamsSourceAttribute.cs index 570be32179..3587907d64 100644 --- a/src/BenchmarkDotNet.Annotations/Attributes/ParamsSourceAttribute.cs +++ b/src/BenchmarkDotNet.Annotations/Attributes/ParamsSourceAttribute.cs @@ -6,7 +6,18 @@ namespace BenchmarkDotNet.Attributes public class ParamsSourceAttribute : PriorityAttribute { public string Name { get; } + public Type? Type { get; } - public ParamsSourceAttribute(string name) => Name = name; + public ParamsSourceAttribute(string name) + { + Name = name; + Type = null; + } + + public ParamsSourceAttribute(Type type, string name) + { + Name = name; + Type = type; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet.Annotations/BenchmarkDotNet.Annotations.csproj b/src/BenchmarkDotNet.Annotations/BenchmarkDotNet.Annotations.csproj index 2e7361e163..ccc51bcd9a 100644 --- a/src/BenchmarkDotNet.Annotations/BenchmarkDotNet.Annotations.csproj +++ b/src/BenchmarkDotNet.Annotations/BenchmarkDotNet.Annotations.csproj @@ -2,7 +2,7 @@ Basic BenchmarkDotNet attributes that can be used to annotate your benchmarks - netstandard1.0;netstandard2.0 + netstandard2.0 $(NoWarn);1701;1702;1705;1591;3005;NU1702;CA1825 BenchmarkDotNet.Annotations BenchmarkDotNet.Annotations diff --git a/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs b/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs index 4fc2db7388..16b242d2dc 100644 --- a/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs +++ b/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs @@ -110,6 +110,11 @@ public enum RuntimeMoniker /// Net90, + /// + /// .NET 10.0 + /// + Net10_0, + /// /// NativeAOT compiled as net6.0 /// @@ -130,6 +135,11 @@ public enum RuntimeMoniker /// NativeAot90, + /// + /// NativeAOT compiled as net10.0 + /// + NativeAot10_0, + /// /// WebAssembly with default .Net version /// @@ -160,6 +170,11 @@ public enum RuntimeMoniker /// WasmNet90, + /// + /// WebAssembly with net10.0 + /// + WasmNet10_0, + /// /// Mono with the Ahead of Time LLVM Compiler backend /// @@ -185,6 +200,11 @@ public enum RuntimeMoniker /// MonoAOTLLVMNet90, + /// + /// Mono with the Ahead of Time LLVM Compiler backend and net10.0 + /// + MonoAOTLLVMNet10_0, + /// /// .NET 6 using MonoVM (not CLR which is the default) /// @@ -204,5 +224,10 @@ public enum RuntimeMoniker /// .NET 9 using MonoVM (not CLR which is the default) /// Mono90, + + /// + /// .NET 10 using MonoVM (not CLR which is the default) + /// + Mono10_0, } } diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/BenchmarkDotNet.Diagnostics.Windows.csproj b/src/BenchmarkDotNet.Diagnostics.Windows/BenchmarkDotNet.Diagnostics.Windows.csproj index 6503c7fa7b..e2eb3d2b68 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/BenchmarkDotNet.Diagnostics.Windows.csproj +++ b/src/BenchmarkDotNet.Diagnostics.Windows/BenchmarkDotNet.Diagnostics.Windows.csproj @@ -12,6 +12,6 @@ - + diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs index 80423d72cf..4205195dc5 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs @@ -8,6 +8,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; using Microsoft.Diagnostics.Tracing; using Microsoft.Diagnostics.Tracing.Parsers; @@ -15,7 +16,7 @@ namespace BenchmarkDotNet.Diagnostics.Windows { - public abstract class EtwDiagnoser where TStats : new() + public abstract class EtwDiagnoser : DisposeAtProcessTermination where TStats : new() { internal readonly LogCapture Logger = new LogCapture(); protected readonly Dictionary BenchmarkToProcess = new Dictionary(); @@ -39,11 +40,6 @@ protected void Start(DiagnoserActionParameters parameters) BenchmarkToProcess.Add(parameters.BenchmarkCase, parameters.Process.Id); StatsPerProcess.TryAdd(parameters.Process.Id, GetInitializedStats(parameters)); - // Important: Must wire-up clean-up events prior to acquiring IDisposable instance (Session property) - // This is in effect the inverted sequence of actions in the Stop() method. - Console.CancelKeyPress += OnConsoleCancelKeyPress; - AppDomain.CurrentDomain.ProcessExit += OnProcessExit; - Session = CreateSession(parameters.BenchmarkCase); EnableProvider(); @@ -80,11 +76,13 @@ protected virtual void EnableProvider() protected void Stop() { WaitForDelayedEvents(); + Dispose(); + } - Session.Dispose(); - - Console.CancelKeyPress -= OnConsoleCancelKeyPress; - AppDomain.CurrentDomain.ProcessExit -= OnProcessExit; + public override void Dispose() + { + Session?.Dispose(); + base.Dispose(); } private void Clear() @@ -93,10 +91,6 @@ private void Clear() StatsPerProcess.Clear(); } - private void OnConsoleCancelKeyPress(object sender, ConsoleCancelEventArgs e) => Session?.Dispose(); - - private void OnProcessExit(object sender, EventArgs e) => Session?.Dispose(); - private static string GetSessionName(string prefix, BenchmarkCase benchmarkCase, ParameterInstances? parameters = null) { if (parameters != null && parameters.Items.Count > 0) diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs b/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs index 6641dd5a45..57f57d9e1a 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs @@ -120,21 +120,10 @@ private void Start(DiagnoserActionParameters parameters) private void Stop(DiagnoserActionParameters parameters) { WaitForDelayedEvents(); - string userSessionFile; - try - { - kernelSession.Stop(); - heapSession?.Stop(); - userSession.Stop(); - - userSessionFile = userSession.FilePath; - } - finally - { - kernelSession.Dispose(); - heapSession?.Dispose(); - userSession.Dispose(); - } + string userSessionFile = userSession.FilePath; + kernelSession.Dispose(); + heapSession?.Dispose(); + userSession.Dispose(); // Merge the 'primary' etl file X.etl (userSession) with any files that match .clr*.etl .user*.etl. and .kernel.etl. TraceEventSession.MergeInPlace(userSessionFile, TextWriter.Null); diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/HardwareCounters.cs b/src/BenchmarkDotNet.Diagnostics.Windows/HardwareCounters.cs index 8b3c46b4d5..d159e72a65 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/HardwareCounters.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/HardwareCounters.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Toolchains.InProcess.Emit; @@ -31,7 +32,7 @@ private static readonly Dictionary EtwTranslations public static IEnumerable Validate(ValidationParameters validationParameters, bool mandatory) { - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) { yield return new ValidationError(true, "Hardware Counters and EtwProfiler are supported only on Windows"); yield break; diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs index b189af0acb..0d06faf75b 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Loggers; @@ -33,7 +34,7 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) public IEnumerable Validate(ValidationParameters validationParameters) { - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) { yield return new ValidationError(true, $"{GetType().Name} is supported only on Windows"); } diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/JitStatsDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/JitStatsDiagnoser.cs index a86989aef5..b5d8a9c359 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/JitStatsDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/JitStatsDiagnoser.cs @@ -6,6 +6,7 @@ using BenchmarkDotNet.Running; using Microsoft.Diagnostics.Tracing.Parsers; using Microsoft.Diagnostics.Tracing.Session; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Diagnostics.Windows { @@ -59,7 +60,7 @@ protected override void AttachToEvents(TraceEventSession session, BenchmarkCase private sealed class MethodsJittedDescriptor : IMetricDescriptor { - internal static readonly MethodsJittedDescriptor Instance = new (); + internal static readonly MethodsJittedDescriptor Instance = new(); public string Id => nameof(MethodsJittedDescriptor); public string DisplayName => "Methods JITted"; @@ -74,7 +75,7 @@ private sealed class MethodsJittedDescriptor : IMetricDescriptor private sealed class MethodsTieredDescriptor : IMetricDescriptor { - internal static readonly MethodsTieredDescriptor Instance = new (); + internal static readonly MethodsTieredDescriptor Instance = new(); public string Id => nameof(MethodsTieredDescriptor); public string DisplayName => "Methods Tiered"; @@ -89,7 +90,7 @@ private sealed class MethodsTieredDescriptor : IMetricDescriptor private sealed class JitAllocatedMemoryDescriptor : IMetricDescriptor { - internal static readonly JitAllocatedMemoryDescriptor Instance = new (); + internal static readonly JitAllocatedMemoryDescriptor Instance = new(); public string Id => nameof(JitAllocatedMemoryDescriptor); public string DisplayName => "JIT allocated memory"; @@ -97,7 +98,7 @@ private sealed class JitAllocatedMemoryDescriptor : IMetricDescriptor public bool TheGreaterTheBetter => false; public string NumberFormat => "N0"; public UnitType UnitType => UnitType.Size; - public string Unit => SizeUnit.B.Name; + public string Unit => SizeUnit.B.Abbreviation; public int PriorityInCategory => 0; public bool GetIsAvailable(Metric metric) => true; } diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/Properties/AssemblyInfo.cs b/src/BenchmarkDotNet.Diagnostics.Windows/Properties/AssemblyInfo.cs index a710492849..bf31db808e 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/Properties/AssemblyInfo.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/Properties/AssemblyInfo.cs @@ -7,8 +7,4 @@ [assembly: CLSCompliant(true)] -#if RELEASE [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] -#else -[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests")] -#endif \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/Sessions.cs b/src/BenchmarkDotNet.Diagnostics.Windows/Sessions.cs index cf0b6fa88f..f0f5dcc475 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/Sessions.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/Sessions.cs @@ -5,13 +5,13 @@ using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Running; using Microsoft.Diagnostics.Tracing; using Microsoft.Diagnostics.Tracing.Parsers; using Microsoft.Diagnostics.Tracing.Session; -using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Diagnostics.Windows { @@ -90,7 +90,7 @@ internal override Session EnableProviders() } } - internal abstract class Session : IDisposable + internal abstract class Session : DisposeAtProcessTermination { private const int MaxSessionNameLength = 128; @@ -114,27 +114,16 @@ protected Session(string sessionName, DiagnoserActionParameters details, EtwProf BufferSizeMB = config.BufferSizeInMb, CpuSampleIntervalMSec = config.CpuSampleIntervalInMilliseconds, }; - - Console.CancelKeyPress += OnConsoleCancelKeyPress; - AppDomain.CurrentDomain.ProcessExit += OnProcessExit; } - public void Dispose() => TraceEventSession.Dispose(); - - internal void Stop() + public override void Dispose() { - TraceEventSession.Stop(); - - Console.CancelKeyPress -= OnConsoleCancelKeyPress; - AppDomain.CurrentDomain.ProcessExit -= OnProcessExit; + TraceEventSession.Dispose(); + base.Dispose(); } internal abstract Session EnableProviders(); - private void OnConsoleCancelKeyPress(object sender, ConsoleCancelEventArgs e) => Stop(); - - private void OnProcessExit(object sender, EventArgs e) => Stop(); - protected static string GetSessionName(BenchmarkCase benchmarkCase) { string benchmarkName = FullNameProvider.GetBenchmarkName(benchmarkCase); diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs index 76061eed08..295ea1d46b 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; -using BenchmarkDotNet.Columns; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Extensions; @@ -12,6 +11,7 @@ using Microsoft.Diagnostics.Tracing.Etlx; using Microsoft.Diagnostics.Tracing.Parsers.Kernel; using Microsoft.Diagnostics.Tracing.Stacks; +using Perfolizer.Metrology; using Address = System.UInt64; namespace BenchmarkDotNet.Diagnostics.Windows.Tracing @@ -38,7 +38,7 @@ public NativeMemoryLogParser(string etlFilePath, BenchmarkCase benchmarkCase, IL this.benchmarkCase = benchmarkCase; this.logger = logger; - moduleName = programName; + moduleName = programName.ToLowerInvariant(); functionNames = new[] { nameof(EngineParameters.WorkloadActionUnroll), @@ -246,12 +246,14 @@ bool IsCallStackIn(StackSourceCallStackIndex index) var memoryAllocatedPerOperation = totalAllocation / totalOperation; var memoryLeakPerOperation = nativeLeakSize / totalOperation; - logger.WriteLine($"Native memory allocated per single operation: {SizeValue.FromBytes(memoryAllocatedPerOperation).ToString(SizeUnit.B, benchmarkCase.Config.CultureInfo)}"); + logger.WriteLine( + $"Native memory allocated per single operation: {SizeValue.FromBytes(memoryAllocatedPerOperation).ToString(SizeUnit.B, null, benchmarkCase.Config.CultureInfo)}"); logger.WriteLine($"Count of allocated object: {countOfAllocatedObject / totalOperation}"); if (nativeLeakSize != 0) { - logger.WriteLine($"Native memory leak per single operation: {SizeValue.FromBytes(memoryLeakPerOperation).ToString(SizeUnit.B, benchmarkCase.Config.CultureInfo)}"); + logger.WriteLine( + $"Native memory leak per single operation: {SizeValue.FromBytes(memoryLeakPerOperation).ToString(SizeUnit.B, null, benchmarkCase.Config.CultureInfo)}"); } var heapInfoList = heaps.Select(h => new { Address = h.Key, h.Value.Count, types = h.Value.Values }); @@ -267,7 +269,8 @@ bool IsCallStackIn(StackSourceCallStackIndex index) }; } - private static Dictionary CreateHeapCache(Address heapHandle, Dictionary> heaps, ref Dictionary lastHeapAllocs, ref Address lastHeapHandle) + private static Dictionary CreateHeapCache(Address heapHandle, Dictionary> heaps, + ref Dictionary lastHeapAllocs, ref Address lastHeapHandle) { Dictionary ret; @@ -282,4 +285,4 @@ private static Dictionary CreateHeapCache(Address heapHandle, Dic return ret; } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotMemory/BenchmarkDotNet.Diagnostics.dotMemory.csproj b/src/BenchmarkDotNet.Diagnostics.dotMemory/BenchmarkDotNet.Diagnostics.dotMemory.csproj new file mode 100644 index 0000000000..cc209f4258 --- /dev/null +++ b/src/BenchmarkDotNet.Diagnostics.dotMemory/BenchmarkDotNet.Diagnostics.dotMemory.csproj @@ -0,0 +1,20 @@ + + + + net6.0;net462;netcoreapp3.1 + $(NoWarn);1591 + BenchmarkDotNet.Diagnostics.dotMemory + BenchmarkDotNet.Diagnostics.dotMemory + BenchmarkDotNet.Diagnostics.dotMemory + enable + + + + + + + + + + + diff --git a/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoser.cs new file mode 100644 index 0000000000..7616ca468c --- /dev/null +++ b/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoser.cs @@ -0,0 +1,126 @@ +using System; +using System.Reflection; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Jobs; +using JetBrains.Profiler.SelfApi; + +namespace BenchmarkDotNet.Diagnostics.dotMemory; + +public class DotMemoryDiagnoser(Uri? nugetUrl = null, string? downloadTo = null) : SnapshotProfilerBase +{ + public override string ShortName => "dotMemory"; + + protected override void InitTool(Progress progress) + { + DotMemory.InitAsync(progress, nugetUrl, NuGetApi.V3, downloadTo).Wait(); + } + + protected override void AttachToCurrentProcess(string snapshotFile) + { + DotMemory.Attach(new DotMemory.Config().SaveToFile(snapshotFile)); + } + + protected override void AttachToProcessByPid(int pid, string snapshotFile) + { + DotMemory.Attach(new DotMemory.Config().ProfileExternalProcess(pid).SaveToFile(snapshotFile)); + } + + protected override void TakeSnapshot() + { + DotMemory.GetSnapshot(); + } + + protected override void Detach() + { + DotMemory.Detach(); + } + + protected override string CreateSnapshotFilePath(DiagnoserActionParameters parameters) + { + return ArtifactFileNameHelper.GetFilePath(parameters, "snapshots", DateTime.Now, "dmw", ".0000".Length); + } + + protected override string GetRunnerPath() + { + var consoleRunnerPackageField = typeof(DotMemory).GetField("ConsoleRunnerPackage", BindingFlags.NonPublic | BindingFlags.Static); + if (consoleRunnerPackageField == null) + throw new InvalidOperationException("Field 'ConsoleRunnerPackage' not found."); + + object? consoleRunnerPackage = consoleRunnerPackageField.GetValue(null); + if (consoleRunnerPackage == null) + throw new InvalidOperationException("Unable to get value of 'ConsoleRunnerPackage'."); + + var consoleRunnerPackageType = consoleRunnerPackage.GetType(); + var getRunnerPathMethod = consoleRunnerPackageType.GetMethod("GetRunnerPath"); + if (getRunnerPathMethod == null) + throw new InvalidOperationException("Method 'GetRunnerPath' not found."); + + string? runnerPath = getRunnerPathMethod.Invoke(consoleRunnerPackage, null) as string; + if (runnerPath == null) + throw new InvalidOperationException("Unable to invoke 'GetRunnerPath'."); + + return runnerPath; + } + + internal override bool IsSupported(RuntimeMoniker runtimeMoniker) + { + switch (runtimeMoniker) + { + case RuntimeMoniker.HostProcess: + case RuntimeMoniker.Net461: + case RuntimeMoniker.Net462: + case RuntimeMoniker.Net47: + case RuntimeMoniker.Net471: + case RuntimeMoniker.Net472: + case RuntimeMoniker.Net48: + case RuntimeMoniker.Net481: + case RuntimeMoniker.Net50: + case RuntimeMoniker.Net60: + case RuntimeMoniker.Net70: + case RuntimeMoniker.Net80: + case RuntimeMoniker.Net90: + case RuntimeMoniker.Net10_0: + return true; + case RuntimeMoniker.NotRecognized: + case RuntimeMoniker.Mono: + case RuntimeMoniker.NativeAot60: + case RuntimeMoniker.NativeAot70: + case RuntimeMoniker.NativeAot80: + case RuntimeMoniker.NativeAot90: + case RuntimeMoniker.NativeAot10_0: + case RuntimeMoniker.Wasm: + case RuntimeMoniker.WasmNet50: + case RuntimeMoniker.WasmNet60: + case RuntimeMoniker.WasmNet70: + case RuntimeMoniker.WasmNet80: + case RuntimeMoniker.WasmNet90: + case RuntimeMoniker.WasmNet10_0: + case RuntimeMoniker.MonoAOTLLVM: + case RuntimeMoniker.MonoAOTLLVMNet60: + case RuntimeMoniker.MonoAOTLLVMNet70: + case RuntimeMoniker.MonoAOTLLVMNet80: + case RuntimeMoniker.MonoAOTLLVMNet90: + case RuntimeMoniker.MonoAOTLLVMNet10_0: + case RuntimeMoniker.Mono60: + case RuntimeMoniker.Mono70: + case RuntimeMoniker.Mono80: + case RuntimeMoniker.Mono90: + case RuntimeMoniker.Mono10_0: +#pragma warning disable CS0618 // Type or member is obsolete + case RuntimeMoniker.NetCoreApp50: +#pragma warning restore CS0618 // Type or member is obsolete + return false; + case RuntimeMoniker.NetCoreApp20: + case RuntimeMoniker.NetCoreApp21: + case RuntimeMoniker.NetCoreApp22: + return OsDetector.IsWindows(); + case RuntimeMoniker.NetCoreApp30: + case RuntimeMoniker.NetCoreApp31: + return OsDetector.IsWindows() || OsDetector.IsLinux(); + default: + throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, $"Runtime moniker {runtimeMoniker} is not supported"); + } + } +} diff --git a/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoserAttribute.cs b/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoserAttribute.cs new file mode 100644 index 0000000000..c0fb55d4f1 --- /dev/null +++ b/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoserAttribute.cs @@ -0,0 +1,22 @@ +using System; +using BenchmarkDotNet.Configs; + +namespace BenchmarkDotNet.Diagnostics.dotMemory; + +[AttributeUsage(AttributeTargets.Class)] +public class DotMemoryDiagnoserAttribute : Attribute, IConfigSource +{ + public IConfig Config { get; } + + public DotMemoryDiagnoserAttribute() + { + var diagnoser = new DotMemoryDiagnoser(); + Config = ManualConfig.CreateEmpty().AddDiagnoser(diagnoser); + } + + public DotMemoryDiagnoserAttribute(Uri? nugetUrl, string? downloadTo = null) + { + var diagnoser = new DotMemoryDiagnoser(nugetUrl, downloadTo); + Config = ManualConfig.CreateEmpty().AddDiagnoser(diagnoser); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotMemory/Properties/AssemblyInfo.cs b/src/BenchmarkDotNet.Diagnostics.dotMemory/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..4cc109a0a4 --- /dev/null +++ b/src/BenchmarkDotNet.Diagnostics.dotMemory/Properties/AssemblyInfo.cs @@ -0,0 +1,7 @@ +using System; +using System.Runtime.CompilerServices; +using BenchmarkDotNet.Properties; + +[assembly: CLSCompliant(true)] + +[assembly: InternalsVisibleTo("BenchmarkDotNet.Tests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/BenchmarkDotNet.Diagnostics.dotTrace.csproj b/src/BenchmarkDotNet.Diagnostics.dotTrace/BenchmarkDotNet.Diagnostics.dotTrace.csproj index e7852f1575..8206182d6a 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/BenchmarkDotNet.Diagnostics.dotTrace.csproj +++ b/src/BenchmarkDotNet.Diagnostics.dotTrace/BenchmarkDotNet.Diagnostics.dotTrace.csproj @@ -14,7 +14,7 @@ - + diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoser.cs index fe9e77005a..8b5d3a858f 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoser.cs @@ -1,151 +1,129 @@ using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using BenchmarkDotNet.Analysers; +using System.Reflection; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains; -using BenchmarkDotNet.Validators; -using RunMode = BenchmarkDotNet.Diagnosers.RunMode; +using JetBrains.Profiler.SelfApi; -namespace BenchmarkDotNet.Diagnostics.dotTrace +namespace BenchmarkDotNet.Diagnostics.dotTrace; + +public class DotTraceDiagnoser(Uri? nugetUrl = null, string? downloadTo = null) : SnapshotProfilerBase { - public class DotTraceDiagnoser : IProfiler + public override string ShortName => "dotTrace"; + + protected override void InitTool(Progress progress) { - private readonly Uri? nugetUrl; - private readonly string? toolsDownloadFolder; + DotTrace.InitAsync(progress, nugetUrl, NuGetApi.V3, downloadTo).Wait(); + } - public DotTraceDiagnoser(Uri? nugetUrl = null, string? toolsDownloadFolder = null) - { - this.nugetUrl = nugetUrl; - this.toolsDownloadFolder = toolsDownloadFolder; - } + protected override void AttachToCurrentProcess(string snapshotFile) + { + DotTrace.Attach(new DotTrace.Config().SaveToFile(snapshotFile)); + DotTrace.StartCollectingData(); + } - public IEnumerable Ids => new[] { "DotTrace" }; - public string ShortName => "dotTrace"; + protected override void AttachToProcessByPid(int pid, string snapshotFile) + { + DotTrace.Attach(new DotTrace.Config().ProfileExternalProcess(pid).SaveToFile(snapshotFile)); + DotTrace.StartCollectingData(); + } - public RunMode GetRunMode(BenchmarkCase benchmarkCase) - { - return IsSupported(benchmarkCase.Job.Environment.GetRuntime().RuntimeMoniker) ? RunMode.ExtraRun : RunMode.None; - } + protected override void TakeSnapshot() + { + DotTrace.StopCollectingData(); + DotTrace.SaveData(); + } - private readonly List snapshotFilePaths = new (); + protected override void Detach() + { + DotTrace.Detach(); + } - public void Handle(HostSignal signal, DiagnoserActionParameters parameters) - { - var job = parameters.BenchmarkCase.Job; - bool isInProcess = job.GetToolchain().IsInProcess; - var logger = parameters.Config.GetCompositeLogger(); - DotTraceToolBase tool = isInProcess - ? new InProcessDotTraceTool(logger, nugetUrl, downloadTo: toolsDownloadFolder) - : new ExternalDotTraceTool(logger, nugetUrl, downloadTo: toolsDownloadFolder); + protected override string CreateSnapshotFilePath(DiagnoserActionParameters parameters) + { + return ArtifactFileNameHelper.GetFilePath(parameters, "snapshots", DateTime.Now, "dtp", ".0000".Length); + } - var runtimeMoniker = job.Environment.GetRuntime().RuntimeMoniker; - if (!IsSupported(runtimeMoniker)) - { - logger.WriteLineError($"Runtime '{runtimeMoniker}' is not supported by dotTrace"); - return; - } + protected override string GetRunnerPath() + { + var consoleRunnerPackageField = typeof(DotTrace).GetField("ConsoleRunnerPackage", BindingFlags.NonPublic | BindingFlags.Static); + if (consoleRunnerPackageField == null) + throw new InvalidOperationException("Field 'ConsoleRunnerPackage' not found."); - switch (signal) - { - case HostSignal.BeforeAnythingElse: - tool.Init(parameters); - break; - case HostSignal.BeforeActualRun: - snapshotFilePaths.Add(tool.Start(parameters)); - break; - case HostSignal.AfterActualRun: - tool.Stop(parameters); - break; - } - } + object? consoleRunnerPackage = consoleRunnerPackageField.GetValue(null); + if (consoleRunnerPackage == null) + throw new InvalidOperationException("Unable to get value of 'ConsoleRunnerPackage'."); - public IEnumerable Exporters => Enumerable.Empty(); - public IEnumerable Analysers => Enumerable.Empty(); + var consoleRunnerPackageType = consoleRunnerPackage.GetType(); + var getRunnerPathMethod = consoleRunnerPackageType.GetMethod("GetRunnerPath"); + if (getRunnerPathMethod == null) + throw new InvalidOperationException("Method 'GetRunnerPath' not found."); - public IEnumerable Validate(ValidationParameters validationParameters) - { - var runtimeMonikers = validationParameters.Benchmarks.Select(b => b.Job.Environment.GetRuntime().RuntimeMoniker).Distinct(); - foreach (var runtimeMoniker in runtimeMonikers) - { - if (!IsSupported(runtimeMoniker)) - yield return new ValidationError(true, $"Runtime '{runtimeMoniker}' is not supported by dotTrace"); - } - } + string? runnerPath = getRunnerPathMethod.Invoke(consoleRunnerPackage, null) as string; + if (runnerPath == null) + throw new InvalidOperationException("Unable to invoke 'GetRunnerPath'."); - internal static bool IsSupported(RuntimeMoniker runtimeMoniker) + return runnerPath; + } + + internal override bool IsSupported(RuntimeMoniker runtimeMoniker) + { + switch (runtimeMoniker) { - switch (runtimeMoniker) - { - case RuntimeMoniker.HostProcess: - case RuntimeMoniker.Net461: - case RuntimeMoniker.Net462: - case RuntimeMoniker.Net47: - case RuntimeMoniker.Net471: - case RuntimeMoniker.Net472: - case RuntimeMoniker.Net48: - case RuntimeMoniker.Net481: - case RuntimeMoniker.Net50: - case RuntimeMoniker.Net60: - case RuntimeMoniker.Net70: - case RuntimeMoniker.Net80: - case RuntimeMoniker.Net90: - return true; - case RuntimeMoniker.NotRecognized: - case RuntimeMoniker.Mono: - case RuntimeMoniker.NativeAot60: - case RuntimeMoniker.NativeAot70: - case RuntimeMoniker.NativeAot80: - case RuntimeMoniker.NativeAot90: - case RuntimeMoniker.Wasm: - case RuntimeMoniker.WasmNet50: - case RuntimeMoniker.WasmNet60: - case RuntimeMoniker.WasmNet70: - case RuntimeMoniker.WasmNet80: - case RuntimeMoniker.WasmNet90: - case RuntimeMoniker.MonoAOTLLVM: - case RuntimeMoniker.MonoAOTLLVMNet60: - case RuntimeMoniker.MonoAOTLLVMNet70: - case RuntimeMoniker.MonoAOTLLVMNet80: - case RuntimeMoniker.MonoAOTLLVMNet90: - case RuntimeMoniker.Mono60: - case RuntimeMoniker.Mono70: - case RuntimeMoniker.Mono80: - case RuntimeMoniker.Mono90: + case RuntimeMoniker.HostProcess: + case RuntimeMoniker.Net461: + case RuntimeMoniker.Net462: + case RuntimeMoniker.Net47: + case RuntimeMoniker.Net471: + case RuntimeMoniker.Net472: + case RuntimeMoniker.Net48: + case RuntimeMoniker.Net481: + case RuntimeMoniker.Net50: + case RuntimeMoniker.Net60: + case RuntimeMoniker.Net70: + case RuntimeMoniker.Net80: + case RuntimeMoniker.Net90: + case RuntimeMoniker.Net10_0: + return true; + case RuntimeMoniker.NotRecognized: + case RuntimeMoniker.Mono: + case RuntimeMoniker.NativeAot60: + case RuntimeMoniker.NativeAot70: + case RuntimeMoniker.NativeAot80: + case RuntimeMoniker.NativeAot90: + case RuntimeMoniker.NativeAot10_0: + case RuntimeMoniker.Wasm: + case RuntimeMoniker.WasmNet50: + case RuntimeMoniker.WasmNet60: + case RuntimeMoniker.WasmNet70: + case RuntimeMoniker.WasmNet80: + case RuntimeMoniker.WasmNet90: + case RuntimeMoniker.WasmNet10_0: + case RuntimeMoniker.MonoAOTLLVM: + case RuntimeMoniker.MonoAOTLLVMNet60: + case RuntimeMoniker.MonoAOTLLVMNet70: + case RuntimeMoniker.MonoAOTLLVMNet80: + case RuntimeMoniker.MonoAOTLLVMNet90: + case RuntimeMoniker.MonoAOTLLVMNet10_0: + case RuntimeMoniker.Mono60: + case RuntimeMoniker.Mono70: + case RuntimeMoniker.Mono80: + case RuntimeMoniker.Mono90: + case RuntimeMoniker.Mono10_0: #pragma warning disable CS0618 // Type or member is obsolete - case RuntimeMoniker.NetCoreApp50: + case RuntimeMoniker.NetCoreApp50: #pragma warning restore CS0618 // Type or member is obsolete - return false; - case RuntimeMoniker.NetCoreApp20: - case RuntimeMoniker.NetCoreApp21: - case RuntimeMoniker.NetCoreApp22: - return RuntimeInformation.IsWindows(); - case RuntimeMoniker.NetCoreApp30: - case RuntimeMoniker.NetCoreApp31: - return RuntimeInformation.IsWindows() || RuntimeInformation.IsLinux(); - default: - throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, $"Runtime moniker {runtimeMoniker} is not supported"); - } - } - - public IEnumerable ProcessResults(DiagnoserResults results) => ImmutableArray.Empty; - - public void DisplayResults(ILogger logger) - { - if (snapshotFilePaths.Any()) - { - logger.WriteLineInfo("The following dotTrace snapshots were generated:"); - foreach (string snapshotFilePath in snapshotFilePaths) - logger.WriteLineInfo($"* {snapshotFilePath}"); - } + return false; + case RuntimeMoniker.NetCoreApp20: + case RuntimeMoniker.NetCoreApp21: + case RuntimeMoniker.NetCoreApp22: + return OsDetector.IsWindows(); + case RuntimeMoniker.NetCoreApp30: + case RuntimeMoniker.NetCoreApp31: + return OsDetector.IsWindows() || OsDetector.IsLinux(); + default: + throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, $"Runtime moniker {runtimeMoniker} is not supported"); } } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoserAttribute.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoserAttribute.cs index 19e88a6de1..f056a98cbd 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoserAttribute.cs +++ b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoserAttribute.cs @@ -1,21 +1,22 @@ using System; using BenchmarkDotNet.Configs; -namespace BenchmarkDotNet.Diagnostics.dotTrace +namespace BenchmarkDotNet.Diagnostics.dotTrace; + +[AttributeUsage(AttributeTargets.Class)] +public class DotTraceDiagnoserAttribute : Attribute, IConfigSource { - [AttributeUsage(AttributeTargets.Class)] - public class DotTraceDiagnoserAttribute : Attribute, IConfigSource - { - public IConfig Config { get; } + public IConfig Config { get; } - public DotTraceDiagnoserAttribute() - { - Config = ManualConfig.CreateEmpty().AddDiagnoser(new DotTraceDiagnoser()); - } + public DotTraceDiagnoserAttribute() + { + var diagnoser = new DotTraceDiagnoser(); + Config = ManualConfig.CreateEmpty().AddDiagnoser(diagnoser); + } - public DotTraceDiagnoserAttribute(Uri? nugetUrl = null, string? toolsDownloadFolder = null) - { - Config = ManualConfig.CreateEmpty().AddDiagnoser(new DotTraceDiagnoser(nugetUrl, toolsDownloadFolder)); - } + public DotTraceDiagnoserAttribute(Uri? nugetUrl, string? downloadTo = null) + { + var diagnoser = new DotTraceDiagnoser(nugetUrl, downloadTo); + Config = ManualConfig.CreateEmpty().AddDiagnoser(diagnoser); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceToolBase.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceToolBase.cs deleted file mode 100644 index f2f07625fa..0000000000 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceToolBase.cs +++ /dev/null @@ -1,145 +0,0 @@ -using System; -using System.IO; -using System.Reflection; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Loggers; -using JetBrains.Profiler.SelfApi; - -namespace BenchmarkDotNet.Diagnostics.dotTrace -{ - internal abstract class DotTraceToolBase - { - private readonly ILogger logger; - private readonly Uri? nugetUrl; - private readonly NuGetApi nugetApi; - private readonly string? downloadTo; - - protected DotTraceToolBase(ILogger logger, Uri? nugetUrl = null, NuGetApi nugetApi = NuGetApi.V3, string? downloadTo = null) - { - this.logger = logger; - this.nugetUrl = nugetUrl; - this.nugetApi = nugetApi; - this.downloadTo = downloadTo; - } - - public void Init(DiagnoserActionParameters parameters) - { - try - { - logger.WriteLineInfo("Ensuring that dotTrace prerequisite is installed..."); - var progress = new Progress(logger, "Installing DotTrace"); - DotTrace.EnsurePrerequisiteAsync(progress, nugetUrl, nugetApi, downloadTo).Wait(); - logger.WriteLineInfo("dotTrace prerequisite is installed"); - logger.WriteLineInfo($"dotTrace runner path: {GetRunnerPath()}"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - } - } - - protected abstract bool AttachOnly { get; } - protected abstract void Attach(DiagnoserActionParameters parameters, string snapshotFile); - protected abstract void StartCollectingData(); - protected abstract void SaveData(); - protected abstract void Detach(); - - public string Start(DiagnoserActionParameters parameters) - { - string snapshotFile = ArtifactFileNameHelper.GetFilePath(parameters, "snapshots", DateTime.Now, "dtp", ".0000".Length); - string? snapshotDirectory = Path.GetDirectoryName(snapshotFile); - logger.WriteLineInfo($"Target snapshot file: {snapshotFile}"); - if (!Directory.Exists(snapshotDirectory) && snapshotDirectory != null) - { - try - { - Directory.CreateDirectory(snapshotDirectory); - } - catch (Exception e) - { - logger.WriteLineError($"Failed to create directory: {snapshotDirectory}"); - logger.WriteLineError(e.ToString()); - } - } - - try - { - logger.WriteLineInfo("Attaching dotTrace to the process..."); - Attach(parameters, snapshotFile); - logger.WriteLineInfo("dotTrace is successfully attached"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - return snapshotFile; - } - - if (!AttachOnly) - { - try - { - logger.WriteLineInfo("Start collecting data using dataTrace..."); - StartCollectingData(); - logger.WriteLineInfo("Data collecting is successfully started"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - } - } - - return snapshotFile; - } - - public void Stop(DiagnoserActionParameters parameters) - { - if (!AttachOnly) - { - try - { - logger.WriteLineInfo("Saving dotTrace snapshot..."); - SaveData(); - logger.WriteLineInfo("dotTrace snapshot is successfully saved to the artifact folder"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - } - - try - { - logger.WriteLineInfo("Detaching dotTrace from the process..."); - Detach(); - logger.WriteLineInfo("dotTrace is successfully detached"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - } - } - } - - protected string GetRunnerPath() - { - var consoleRunnerPackageField = typeof(DotTrace).GetField("ConsoleRunnerPackage", BindingFlags.NonPublic | BindingFlags.Static); - if (consoleRunnerPackageField == null) - throw new InvalidOperationException("Field 'ConsoleRunnerPackage' not found."); - - object? consoleRunnerPackage = consoleRunnerPackageField.GetValue(null); - if (consoleRunnerPackage == null) - throw new InvalidOperationException("Unable to get value of 'ConsoleRunnerPackage'."); - - var consoleRunnerPackageType = consoleRunnerPackage.GetType(); - var getRunnerPathMethod = consoleRunnerPackageType.GetMethod("GetRunnerPath"); - if (getRunnerPathMethod == null) - throw new InvalidOperationException("Method 'GetRunnerPath' not found."); - - string? runnerPath = getRunnerPathMethod.Invoke(consoleRunnerPackage, null) as string; - if (runnerPath == null) - throw new InvalidOperationException("Unable to invoke 'GetRunnerPath'."); - - return runnerPath; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/ExternalDotTraceTool.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/ExternalDotTraceTool.cs deleted file mode 100644 index dfc9903b82..0000000000 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/ExternalDotTraceTool.cs +++ /dev/null @@ -1,84 +0,0 @@ -using System; -using System.Diagnostics; -using System.Threading.Tasks; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Loggers; -using JetBrains.Profiler.SelfApi; -using ILogger = BenchmarkDotNet.Loggers.ILogger; - -namespace BenchmarkDotNet.Diagnostics.dotTrace -{ - internal class ExternalDotTraceTool : DotTraceToolBase - { - private static readonly TimeSpan AttachTimeout = TimeSpan.FromMinutes(5); - - public ExternalDotTraceTool(ILogger logger, Uri? nugetUrl = null, NuGetApi nugetApi = NuGetApi.V3, string? downloadTo = null) : - base(logger, nugetUrl, nugetApi, downloadTo) { } - - protected override bool AttachOnly => true; - - protected override void Attach(DiagnoserActionParameters parameters, string snapshotFile) - { - var logger = parameters.Config.GetCompositeLogger(); - - string runnerPath = GetRunnerPath(); - int pid = parameters.Process.Id; - string arguments = $"attach {pid} --save-to=\"{snapshotFile}\" --service-output=on"; - - logger.WriteLineInfo($"Starting process: '{runnerPath} {arguments}'"); - - var processStartInfo = new ProcessStartInfo - { - FileName = runnerPath, - WorkingDirectory = "", - Arguments = arguments, - UseShellExecute = false, - CreateNoWindow = true, - RedirectStandardOutput = true, - RedirectStandardError = true - }; - - var attachWaitingTask = new TaskCompletionSource(); - var process = new Process { StartInfo = processStartInfo }; - try - { - process.OutputDataReceived += (_, args) => - { - string? content = args.Data; - if (content != null) - { - logger.WriteLineInfo("[dotTrace] " + content); - if (content.Contains("##dotTrace[\"started\"")) - attachWaitingTask.TrySetResult(true); - } - }; - process.ErrorDataReceived += (_, args) => - { - string? content = args.Data; - if (content != null) - logger.WriteLineError("[dotTrace] " + args.Data); - }; - process.Exited += (_, _) => { attachWaitingTask.TrySetResult(false); }; - process.Start(); - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - } - catch (Exception e) - { - attachWaitingTask.TrySetResult(false); - logger.WriteLineError(e.ToString()); - } - - if (!attachWaitingTask.Task.Wait(AttachTimeout)) - throw new Exception($"Failed to attach dotTrace to the target process (timeout: {AttachTimeout.TotalSeconds} sec"); - if (!attachWaitingTask.Task.Result) - throw new Exception($"Failed to attach dotTrace to the target process (ExitCode={process.ExitCode})"); - } - - protected override void StartCollectingData() { } - - protected override void SaveData() { } - - protected override void Detach() { } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/InProcessDotTraceTool.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/InProcessDotTraceTool.cs deleted file mode 100644 index a02c9c1995..0000000000 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/InProcessDotTraceTool.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Loggers; -using JetBrains.Profiler.SelfApi; - -namespace BenchmarkDotNet.Diagnostics.dotTrace -{ - internal class InProcessDotTraceTool : DotTraceToolBase - { - public InProcessDotTraceTool(ILogger logger, Uri? nugetUrl = null, NuGetApi nugetApi = NuGetApi.V3, string? downloadTo = null) : - base(logger, nugetUrl, nugetApi, downloadTo) { } - - protected override bool AttachOnly => false; - - protected override void Attach(DiagnoserActionParameters parameters, string snapshotFile) - { - var config = new DotTrace.Config(); - config.SaveToFile(snapshotFile); - DotTrace.Attach(config); - } - - protected override void StartCollectingData() => DotTrace.StartCollectingData(); - - protected override void SaveData() => DotTrace.SaveData(); - - protected override void Detach() => DotTrace.Detach(); - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/Progress.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/Progress.cs deleted file mode 100644 index c353939f1f..0000000000 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/Progress.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System; -using System.Diagnostics; -using BenchmarkDotNet.Loggers; - -namespace BenchmarkDotNet.Diagnostics.dotTrace -{ - public class Progress : IProgress - { - private static readonly TimeSpan ReportInterval = TimeSpan.FromSeconds(0.1); - - private readonly ILogger logger; - private readonly string title; - - public Progress(ILogger logger, string title) - { - this.logger = logger; - this.title = title; - } - - private int lastProgress; - private Stopwatch? stopwatch; - - public void Report(double value) - { - int progress = (int)Math.Floor(value); - bool needToReport = stopwatch == null || - (stopwatch != null && stopwatch?.Elapsed > ReportInterval) || - progress == 100; - - if (lastProgress != progress && needToReport) - { - logger.WriteLineInfo($"{title}: {progress}%"); - lastProgress = progress; - stopwatch = Stopwatch.StartNew(); - } - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/Properties/AssemblyInfo.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/Properties/AssemblyInfo.cs index 270fdc2c9c..4cc109a0a4 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/Properties/AssemblyInfo.cs +++ b/src/BenchmarkDotNet.Diagnostics.dotTrace/Properties/AssemblyInfo.cs @@ -4,8 +4,4 @@ [assembly: CLSCompliant(true)] -#if RELEASE [assembly: InternalsVisibleTo("BenchmarkDotNet.Tests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] -#else -[assembly: InternalsVisibleTo("BenchmarkDotNet.Tests")] -#endif \ No newline at end of file diff --git a/src/BenchmarkDotNet.Disassembler.x64/BenchmarkDotNet.Disassembler.x64.csproj b/src/BenchmarkDotNet.Disassembler.x64/BenchmarkDotNet.Disassembler.x64.csproj index 3162c74e5d..2f15efcc13 100644 --- a/src/BenchmarkDotNet.Disassembler.x64/BenchmarkDotNet.Disassembler.x64.csproj +++ b/src/BenchmarkDotNet.Disassembler.x64/BenchmarkDotNet.Disassembler.x64.csproj @@ -15,7 +15,7 @@ BenchmarkDotNet.Disassembler - - + + diff --git a/src/BenchmarkDotNet.Disassembler.x64/ClrMdV1Disassembler.cs b/src/BenchmarkDotNet.Disassembler.x64/ClrMdV1Disassembler.cs index 29ee184df3..734aa470e4 100644 --- a/src/BenchmarkDotNet.Disassembler.x64/ClrMdV1Disassembler.cs +++ b/src/BenchmarkDotNet.Disassembler.x64/ClrMdV1Disassembler.cs @@ -93,6 +93,7 @@ private static DisassembledMethod[] Disassemble(Settings settings, State state) { var result = new List(); + using var sourceCodeProvider = new SourceCodeProvider(); while (state.Todo.Count != 0) { var methodInfo = state.Todo.Dequeue(); @@ -101,7 +102,7 @@ private static DisassembledMethod[] Disassemble(Settings settings, State state) continue; // already handled if (settings.MaxDepth >= methodInfo.Depth) - result.Add(DisassembleMethod(methodInfo, state, settings)); + result.Add(DisassembleMethod(methodInfo, state, settings, sourceCodeProvider)); } return result.ToArray(); @@ -110,7 +111,7 @@ private static DisassembledMethod[] Disassemble(Settings settings, State state) private static bool CanBeDisassembled(ClrMethod method) => !((method.ILOffsetMap is null || method.ILOffsetMap.Length == 0) && (method.HotColdInfo is null || method.HotColdInfo.HotStart == 0 || method.HotColdInfo.HotSize == 0)); - private static DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State state, Settings settings) + private static DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State state, Settings settings, SourceCodeProvider sourceCodeProvider) { var method = methodInfo.Method; @@ -133,7 +134,7 @@ private static DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State var uniqueSourceCodeLines = new HashSet(new SharpComparer()); // for getting C# code we always use the original ILOffsetMap foreach (var map in method.ILOffsetMap.Where(map => map.StartAddress < map.EndAddress && map.ILOffset >= 0).OrderBy(map => map.StartAddress)) - foreach (var sharp in SourceCodeProvider.GetSource(method, map)) + foreach (var sharp in sourceCodeProvider.GetSource(method, map)) uniqueSourceCodeLines.Add(sharp); codes.AddRange(uniqueSourceCodeLines); diff --git a/src/BenchmarkDotNet.Disassembler.x64/SourceCodeProvider.cs b/src/BenchmarkDotNet.Disassembler.x64/SourceCodeProvider.cs index c9ac897570..f827369775 100644 --- a/src/BenchmarkDotNet.Disassembler.x64/SourceCodeProvider.cs +++ b/src/BenchmarkDotNet.Disassembler.x64/SourceCodeProvider.cs @@ -8,13 +8,34 @@ namespace BenchmarkDotNet.Disassemblers { - internal static class SourceCodeProvider + // This is taken from the Samples\FileAndLineNumbers projects from microsoft/clrmd, + // and replaces the previously-available SourceLocation functionality. + + internal class SourceLocation { - private static readonly Dictionary SourceFileCache = new Dictionary(); + public string FilePath; + public int LineNumber; + public int LineNumberEnd; + public int ColStart; + public int ColEnd; + } + + internal class SourceCodeProvider : IDisposable + { + private readonly Dictionary sourceFileCache = new Dictionary(); + private readonly Dictionary pdbReaders = new Dictionary(); + + public void Dispose() + { + foreach (var reader in pdbReaders.Values) + { + reader?.Dispose(); + } + } - internal static IEnumerable GetSource(ClrMethod method, ILToNativeMap map) + internal IEnumerable GetSource(ClrMethod method, ILToNativeMap map) { - var sourceLocation = method.GetSourceLocation(map.ILOffset); + var sourceLocation = GetSourceLocation(method, map.ILOffset); if (sourceLocation == null) yield break; @@ -39,16 +60,16 @@ internal static IEnumerable GetSource(ClrMethod method, ILToNativeMap map } } - private static string ReadSourceLine(string file, int line) + private string ReadSourceLine(string file, int line) { - if (!SourceFileCache.TryGetValue(file, out string[] contents)) + if (!sourceFileCache.TryGetValue(file, out string[] contents)) { // sometimes the symbols report some disk location from MS CI machine like "E:\A\_work\308\s\src\mscorlib\shared\System\Random.cs" for .NET Core 2.0 if (!File.Exists(file)) return null; contents = File.ReadAllLines(file); - SourceFileCache.Add(file, contents); + sourceFileCache.Add(file, contents); } return line - 1 < contents.Length @@ -84,29 +105,8 @@ private static string GetSmartPointer(string sourceLine, int? start, int? end) return new string(prefix); } - } - - - // This is taken from the Samples\FileAndLineNumbers projects from microsoft/clrmd, - // and replaces the previously-available SourceLocation functionality. - - internal class SourceLocation - { - public string FilePath; - public int LineNumber; - public int LineNumberEnd; - public int ColStart; - public int ColEnd; - } - - internal static class ClrSourceExtensions - { - // TODO Not sure we want this to be a shared dictionary, especially without - // any synchronization. Probably want to put this hanging off the Context - // somewhere, or inside SymbolCache. - private static readonly Dictionary s_pdbReaders = new Dictionary(); - internal static SourceLocation GetSourceLocation(this ClrMethod method, int ilOffset) + internal SourceLocation GetSourceLocation(ClrMethod method, int ilOffset) { PdbReader reader = GetReaderForMethod(method); if (reader == null) @@ -116,7 +116,7 @@ internal static SourceLocation GetSourceLocation(this ClrMethod method, int ilOf return FindNearestLine(function, ilOffset); } - internal static SourceLocation GetSourceLocation(this ClrStackFrame frame) + internal SourceLocation GetSourceLocation(ClrStackFrame frame) { PdbReader reader = GetReaderForMethod(frame.Method); if (reader == null) @@ -178,7 +178,7 @@ private static int FindIlOffset(ClrStackFrame frame) return last; } - private static PdbReader GetReaderForMethod(ClrMethod method) + private PdbReader GetReaderForMethod(ClrMethod method) { ClrModule module = method?.Type?.Module; PdbInfo info = module?.Pdb; @@ -186,7 +186,7 @@ private static PdbReader GetReaderForMethod(ClrMethod method) PdbReader? reader = null; if (info != null) { - if (!s_pdbReaders.TryGetValue(info, out reader)) + if (!pdbReaders.TryGetValue(info, out reader)) { SymbolLocator locator = GetSymbolLocator(module); string pdbPath = locator.FindPdb(info); @@ -207,7 +207,7 @@ private static PdbReader GetReaderForMethod(ClrMethod method) } } - s_pdbReaders[info] = reader; + pdbReaders[info] = reader; } } diff --git a/src/BenchmarkDotNet.Disassembler.x86/BenchmarkDotNet.Disassembler.x86.csproj b/src/BenchmarkDotNet.Disassembler.x86/BenchmarkDotNet.Disassembler.x86.csproj index 21051d6104..5410f6d77b 100644 --- a/src/BenchmarkDotNet.Disassembler.x86/BenchmarkDotNet.Disassembler.x86.csproj +++ b/src/BenchmarkDotNet.Disassembler.x86/BenchmarkDotNet.Disassembler.x86.csproj @@ -21,7 +21,7 @@ - - + + diff --git a/src/BenchmarkDotNet.Exporters.Plotting/BenchmarkDotNet.Exporters.Plotting.csproj b/src/BenchmarkDotNet.Exporters.Plotting/BenchmarkDotNet.Exporters.Plotting.csproj new file mode 100644 index 0000000000..cefa91a7ca --- /dev/null +++ b/src/BenchmarkDotNet.Exporters.Plotting/BenchmarkDotNet.Exporters.Plotting.csproj @@ -0,0 +1,19 @@ + + + + BenchmarkDotNet plotting export support. + netstandard2.0 + BenchmarkDotNet.Exporters.Plotting + BenchmarkDotNet.Exporters.Plotting + BenchmarkDotNet.Exporters.Plotting + + True + enable + + + + + + + + diff --git a/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs b/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs new file mode 100644 index 0000000000..df43599aec --- /dev/null +++ b/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs @@ -0,0 +1,438 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Properties; +using BenchmarkDotNet.Reports; +using ScottPlot; +using ScottPlot.Plottables; + +namespace BenchmarkDotNet.Exporters.Plotting +{ + /// + /// Provides plot exports as .png files. + /// + public class ScottPlotExporter : IExporter + { + /// + /// Default instance of the exporter with default configuration. + /// + public static readonly IExporter Default = new ScottPlotExporter(); + + /// + /// Gets the name of the Exporter type. + /// + public string Name => nameof(ScottPlotExporter); + + /// + /// Initializes a new instance of ScottPlotExporter. + /// + /// The width of all plots in pixels (optional). Defaults to 1920. + /// The height of all plots in pixels (optional). Defaults to 1080. + public ScottPlotExporter(int width = 1920, int height = 1080) + { + this.Width = width; + this.Height = height; + this.IncludeBarPlot = true; + this.IncludeBoxPlot = true; + this.RotateLabels = true; + } + + /// + /// Gets or sets the width of all plots in pixels. + /// + public int Width { get; set; } + + /// + /// Gets or sets the height of all plots in pixels. + /// + public int Height { get; set; } + + /// + /// Gets or sets the common font size for ticks, labels etc. (defaults to 14). + /// + public int FontSize { get; set; } = 14; + + /// + /// Gets or sets the font size for the chart title. (defaults to 28). + /// + public int TitleFontSize { get; set; } = 28; + + /// + /// Gets or sets a value indicating whether labels for Plot X-axis should be rotated. + /// This allows for longer labels at the expense of chart height. + /// + public bool RotateLabels { get; set; } + + /// + /// Gets or sets a value indicating whether a bar plot for time-per-op + /// measurement values should be exported. + /// + public bool IncludeBarPlot { get; set; } + + /// + /// Gets or sets a value indicating whether a box plot or whisker plot for time-per-op + /// measurement values should be exported. + /// + public bool IncludeBoxPlot { get; set; } + + /// + /// Not supported. + /// + /// This parameter is not used. + /// This parameter is not used. + /// + public void ExportToLog(Summary summary, ILogger logger) + { + throw new NotSupportedException(); + } + + /// + /// Exports plots to .png file. + /// + /// The summary to be exported. + /// Logger to output to. + /// The file paths of every plot exported. + public IEnumerable ExportToFiles(Summary summary, ILogger consoleLogger) + { + var title = summary.Title; + var version = BenchmarkDotNetInfo.Instance.BrandTitle; + var annotations = GetAnnotations(version); + + var (timeUnit, timeScale) = GetTimeUnit(summary.Reports + .SelectMany(m => m.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Result)))); + + foreach (var benchmark in summary.Reports.GroupBy(r => r.BenchmarkCase.Descriptor.Type.Name)) + { + var benchmarkName = benchmark.Key; + + // Get the measurement nanoseconds per op, divided by time scale, grouped by target and Job [param]. + var timeStats = from report in benchmark + let jobId = report.BenchmarkCase.DisplayInfo.Replace(report.BenchmarkCase.Descriptor.DisplayInfo + ": ", string.Empty) + from measurement in report.AllMeasurements + where measurement.Is(IterationMode.Workload, IterationStage.Result) + let measurementValue = measurement.Nanoseconds / measurement.Operations + group measurementValue / timeScale by (Target: report.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo, JobId: jobId) into g + select new ChartStats(g.Key.Target, g.Key.JobId, g.ToList()); + + if (this.IncludeBarPlot) + { + // -barplot.png + yield return CreateBarPlot( + $"{title} - {benchmarkName}", + Path.Combine(summary.ResultsDirectoryPath, $"{title}-{benchmarkName}-barplot.png"), + $"Time ({timeUnit})", + "Target", + timeStats, + annotations); + } + + if (this.IncludeBoxPlot) + { + // -boxplot.png + yield return CreateBoxPlot( + $"{title} - {benchmarkName}", + Path.Combine(summary.ResultsDirectoryPath, $"{title}-{benchmarkName}-boxplot.png"), + $"Time ({timeUnit})", + "Target", + timeStats, + annotations); + } + + /* TODO: Rest of the RPlotExporter plots. + --density.png + --facetTimeline.png + --facetTimelineSmooth.png + ---timelineSmooth.png + ---timelineSmooth.png*/ + } + } + + /// + /// Calculate Standard Deviation. + /// + /// Values to calculate from. + /// Standard deviation of values. + private static double StandardError(IReadOnlyList values) + { + double average = values.Average(); + double sumOfSquaresOfDifferences = values.Select(val => (val - average) * (val - average)).Sum(); + double standardDeviation = Math.Sqrt(sumOfSquaresOfDifferences / values.Count); + return standardDeviation / Math.Sqrt(values.Count); + } + + /// + /// Gets the lowest appropriate time scale across all measurements. + /// + /// All measurements + /// A unit and scaling factor to convert from nanoseconds. + private (string Unit, double ScaleFactor) GetTimeUnit(IEnumerable values) + { + var minValue = values.Select(m => m.Nanoseconds / m.Operations).DefaultIfEmpty(0d).Min(); + if (minValue > 1000000000d) + { + return ("sec", 1000000000d); + } + + if (minValue > 1000000d) + { + return ("ms", 1000000d); + } + + if (minValue > 1000d) + { + return ("us", 1000d); + } + + return ("ns", 1d); + } + + private string CreateBarPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable data, IReadOnlyList annotations) + { + Plot plt = new Plot(); + plt.Title(title, this.TitleFontSize); + plt.YLabel(yLabel, this.FontSize); + plt.XLabel(xLabel, this.FontSize); + + var palette = new ScottPlot.Palettes.Category10(); + + var legendPalette = data.Select(d => d.JobId) + .Distinct() + .Select((jobId, index) => (jobId, index)) + .ToDictionary(t => t.jobId, t => palette.GetColor(t.index)); + + plt.Legend.IsVisible = true; + plt.Legend.Alignment = Alignment.UpperRight; + plt.Legend.FontSize = this.FontSize; + var legend = data.Select(d => d.JobId) + .Distinct() + .Select((label, index) => new LegendItem() + { + LabelText = label, + FillColor = legendPalette[label] + }) + .ToList(); + + plt.Legend.ManualItems.AddRange(legend); + + var jobCount = plt.Legend.ManualItems.Count; + var ticks = data + .Select((d, index) => new Tick(index, d.Target)) + .ToArray(); + + plt.Axes.Left.TickLabelStyle.FontSize = this.FontSize; + plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks); + plt.Axes.Bottom.MajorTickStyle.Length = 0; + plt.Axes.Bottom.TickLabelStyle.FontSize = this.FontSize; + + if (this.RotateLabels) + { + plt.Axes.Bottom.TickLabelStyle.Rotation = 45; + plt.Axes.Bottom.TickLabelStyle.Alignment = Alignment.MiddleLeft; + + // determine the width of the largest tick label + float largestLabelWidth = 0; + foreach (Tick tick in ticks) + { + PixelSize size = plt.Axes.Bottom.TickLabelStyle.Measure(tick.Label).Size; + largestLabelWidth = Math.Max(largestLabelWidth, size.Width); + } + + // ensure axis panels do not get smaller than the largest label + plt.Axes.Bottom.MinimumSize = largestLabelWidth * 2; + plt.Axes.Right.MinimumSize = largestLabelWidth; + } + + var bars = data + .Select((d, index) => new Bar() + { + Position = ticks[index].Position, + Value = d.Mean, + Error = d.StdError, + FillColor = legendPalette[d.JobId] + }); + plt.Add.Bars(bars.ToList()); + + // Tell the plot to autoscale with no padding beneath the bars + plt.Axes.Margins(bottom: 0, right: .2); + + plt.PlottableList.AddRange(annotations); + + plt.SavePng(fileName, this.Width, this.Height); + return Path.GetFullPath(fileName); + } + + private string CreateBoxPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable data, IReadOnlyList annotations) + { + Plot plt = new Plot(); + plt.Title(title, this.TitleFontSize); + plt.YLabel(yLabel, this.FontSize); + plt.XLabel(xLabel, this.FontSize); + + var palette = new ScottPlot.Palettes.Category10(); + + var legendPalette = data.Select(d => d.JobId) + .Distinct() + .Select((jobId, index) => (jobId, index)) + .ToDictionary(t => t.jobId, t => palette.GetColor(t.index)); + + plt.Legend.IsVisible = true; + plt.Legend.Alignment = Alignment.UpperRight; + plt.Legend.FontSize = this.FontSize; + var legend = data.Select(d => d.JobId) + .Distinct() + .Select((label, index) => new LegendItem() + { + LabelText = label, + FillColor = legendPalette[label] + }) + .ToList(); + + plt.Legend.ManualItems.AddRange(legend); + + var jobCount = plt.Legend.ManualItems.Count; + var ticks = data + .Select((d, index) => new Tick(index, d.Target)) + .ToArray(); + + plt.Axes.Left.TickLabelStyle.FontSize = this.FontSize; + plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks); + plt.Axes.Bottom.MajorTickStyle.Length = 0; + plt.Axes.Bottom.TickLabelStyle.FontSize = this.FontSize; + + if (this.RotateLabels) + { + plt.Axes.Bottom.TickLabelStyle.Rotation = 45; + plt.Axes.Bottom.TickLabelStyle.Alignment = Alignment.MiddleLeft; + + // determine the width of the largest tick label + float largestLabelWidth = 0; + foreach (Tick tick in ticks) + { + PixelSize size = plt.Axes.Bottom.TickLabelStyle.Measure(tick.Label).Size; + largestLabelWidth = Math.Max(largestLabelWidth, size.Width); + } + + // ensure axis panels do not get smaller than the largest label + plt.Axes.Bottom.MinimumSize = largestLabelWidth * 2; + plt.Axes.Right.MinimumSize = largestLabelWidth; + } + + int globalIndex = 0; + foreach (var (targetGroup, targetGroupIndex) in data.GroupBy(s => s.Target).Select((targetGroup, index) => (targetGroup, index))) + { + var boxes = targetGroup.Select(job => (job.JobId, Stats: job.CalculateBoxPlotStatistics())).Select((j, jobIndex) => new Box() + { + Position = ticks[globalIndex++].Position, + FillStyle = new FillStyle() { Color = legendPalette[j.JobId] }, + LineStyle = new LineStyle() { Color = Colors.Black }, + BoxMin = j.Stats.Q1, + BoxMax = j.Stats.Q3, + WhiskerMin = j.Stats.Min, + WhiskerMax = j.Stats.Max, + BoxMiddle = j.Stats.Median + }) + .ToList(); + plt.Add.Boxes(boxes); + } + + // Tell the plot to autoscale with a small padding below the boxes. + plt.Axes.Margins(bottom: 0.05, right: .2); + + plt.PlottableList.AddRange(annotations); + + plt.SavePng(fileName, this.Width, this.Height); + return Path.GetFullPath(fileName); + } + + /// + /// Provides a list of annotations to put over the data area. + /// + /// The version to be displayed. + /// A list of annotations for every plot. + private IReadOnlyList GetAnnotations(string version) + { + var versionAnnotation = new Annotation() + { + LabelStyle = + { + Text = version, + FontSize = 14, + ForeColor = new Color(0, 0, 0, 100) + }, + OffsetY = 10, + OffsetX = 20, + Alignment = Alignment.LowerRight + }; + + + return new[] { versionAnnotation }; + } + + private class ChartStats + { + public ChartStats(string Target, string JobId, IReadOnlyList Values) + { + this.Target = Target; + this.JobId = JobId; + this.Values = Values; + } + + public string Target { get; } + + public string JobId { get; } + + public IReadOnlyList Values { get; } + + public double Min => this.Values.DefaultIfEmpty(0d).Min(); + + public double Max => this.Values.DefaultIfEmpty(0d).Max(); + + public double Mean => this.Values.DefaultIfEmpty(0d).Average(); + + public double StdError => StandardError(this.Values); + + + private static (int MidPoint, double Median) CalculateMedian(ReadOnlySpan values) + { + int n = values.Length; + var midPoint = n / 2; + + // Check if count is even, if so use average of the two middle values, + // otherwise take the middle value. + var median = n % 2 == 0 ? (values[midPoint - 1] + values[midPoint]) / 2d : values[midPoint]; + return (midPoint, median); + } + + /// + /// Calculate the mid points. + /// + /// + public (double Min, double Q1, double Median, double Q3, double Max, double[] Outliers) CalculateBoxPlotStatistics() + { + var values = this.Values.ToArray(); + Array.Sort(values); + var s = values.AsSpan(); + var (midPoint, median) = CalculateMedian(s); + + var (q1Index, q1) = midPoint > 0 ? CalculateMedian(s.Slice(0, midPoint)) : (midPoint, median); + var (q3Index, q3) = midPoint + 1 < s.Length ? CalculateMedian(s.Slice(midPoint + 1)) : (midPoint, median); + var iqr = q3 - q1; + var lowerFence = q1 - 1.5d * iqr; + var upperFence = q3 + 1.5d * iqr; + var outliers = values.Where(v => v < lowerFence || v > upperFence).ToArray(); + var nonOutliers = values.Where(v => v >= lowerFence && v <= upperFence).ToArray(); + return ( + nonOutliers.FirstOrDefault(), + q1, + median, + q3, + nonOutliers.LastOrDefault(), + outliers + ); + } + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkCaseExtensions.cs b/src/BenchmarkDotNet.TestAdapter/BenchmarkCaseExtensions.cs new file mode 100644 index 0000000000..82e6d4b6ec --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkCaseExtensions.cs @@ -0,0 +1,96 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Running; +using Microsoft.TestPlatform.AdapterUtilities; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using System; + +namespace BenchmarkDotNet.TestAdapter +{ + /// + /// A set of extensions for BenchmarkCase to support converting to VSTest TestCase objects. + /// + internal static class BenchmarkCaseExtensions + { + /// + /// Converts a BDN BenchmarkCase to a VSTest TestCase. + /// + /// The BenchmarkCase to convert. + /// The dll or exe of the benchmark project. + /// Whether or not the display name should include the job name. + /// The VSTest TestCase. + internal static TestCase ToVsTestCase(this BenchmarkCase benchmarkCase, string assemblyPath, bool includeJobInName = false) + { + var benchmarkMethod = benchmarkCase.Descriptor.WorkloadMethod; + var fullClassName = benchmarkCase.Descriptor.Type.GetCorrectCSharpTypeName(); + var parametrizedMethodName = FullNameProvider.GetMethodName(benchmarkCase); + + var displayJobInfo = benchmarkCase.GetUnrandomizedJobDisplayInfo(); + var displayMethodName = parametrizedMethodName + (includeJobInName ? $" [{displayJobInfo}]" : ""); + var displayName = $"{fullClassName}.{displayMethodName}"; + + // We use displayName as FQN to workaround the Rider/R# problem with FQNs processing + // See: https://github.com/dotnet/BenchmarkDotNet/issues/2494 + var fullyQualifiedName = displayName; + + var vsTestCase = new TestCase(fullyQualifiedName, VsTestAdapter.ExecutorUri, assemblyPath) + { + DisplayName = displayName, + Id = GetTestCaseId(benchmarkCase) + }; + + var benchmarkAttribute = benchmarkMethod.ResolveAttribute(); + if (benchmarkAttribute != null) + { + vsTestCase.CodeFilePath = benchmarkAttribute.SourceCodeFile; + vsTestCase.LineNumber = benchmarkAttribute.SourceCodeLineNumber; + } + + var categories = DefaultCategoryDiscoverer.Instance.GetCategories(benchmarkMethod); + foreach (var category in categories) + vsTestCase.Traits.Add("Category", category); + + vsTestCase.Traits.Add("", "BenchmarkDotNet"); + + return vsTestCase; + } + + /// + /// If an ID is not provided, a random string is used for the ID. This method will identify if randomness was + /// used for the ID and return the Job's DisplayInfo with that randomness removed so that the same benchmark + /// can be referenced across multiple processes. + /// + /// The benchmark case. + /// The benchmark case' job's DisplayInfo without randomness. + internal static string GetUnrandomizedJobDisplayInfo(this BenchmarkCase benchmarkCase) + { + var jobDisplayInfo = benchmarkCase.Job.DisplayInfo; + if (!benchmarkCase.Job.HasValue(CharacteristicObject.IdCharacteristic) && + benchmarkCase.Job.ResolvedId.StartsWith("Job-", StringComparison.OrdinalIgnoreCase)) + { + // Replace Job-ABCDEF with Job + jobDisplayInfo = "Job" + jobDisplayInfo.Substring(benchmarkCase.Job.ResolvedId.Length); + } + + return jobDisplayInfo; + } + + /// + /// Gets an ID for a given BenchmarkCase that is uniquely identifiable from discovery to execution phase. + /// + /// The benchmark case. + /// The test case ID. + internal static Guid GetTestCaseId(this BenchmarkCase benchmarkCase) + { + var testIdProvider = new TestIdProvider(); + testIdProvider.AppendString(VsTestAdapter.ExecutorUriString); + testIdProvider.AppendString(benchmarkCase.Descriptor.Type.Namespace ?? string.Empty); + testIdProvider.AppendString(benchmarkCase.Descriptor.DisplayInfo); + testIdProvider.AppendString(benchmarkCase.GetUnrandomizedJobDisplayInfo()); + testIdProvider.AppendString(benchmarkCase.Parameters.DisplayInfo); + return testIdProvider.GetId(); + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkDotNet.TestAdapter.csproj b/src/BenchmarkDotNet.TestAdapter/BenchmarkDotNet.TestAdapter.csproj new file mode 100644 index 0000000000..5def448d9c --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkDotNet.TestAdapter.csproj @@ -0,0 +1,30 @@ + + + + netstandard2.0;net462 + BenchmarkDotNet.TestAdapter + BenchmarkDotNet.TestAdapter + BenchmarkDotNet.TestAdapter + True + enable + + + + + + + + + + + + + + + + + + + + + diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs b/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs new file mode 100644 index 0000000000..d49c3d24e1 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs @@ -0,0 +1,70 @@ +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using System; +using System.IO; +using System.Linq; +using System.Reflection; + +namespace BenchmarkDotNet.TestAdapter +{ + /// + /// A class used for enumerating all the benchmarks in an assembly. + /// + internal static class BenchmarkEnumerator + { + /// + /// Returns all the BenchmarkRunInfo objects from a given assembly. + /// + /// The dll or exe of the benchmark project. + /// The benchmarks inside the assembly. + public static BenchmarkRunInfo[] GetBenchmarksFromAssemblyPath(string assemblyPath) + { +#if NET462 + // Temporary workaround for BenchmarkDotNet assembly loading issue that occurred under the following conditions: + // 1. Run BenchmarkDotNet.Samples project with following command. + // > dotnet test -c Release --list-tests --framework net462 -tl:off + // 2. When using `BenchmarkDotNet.TestAdapter` package and targeting .NET Framework. + AppDomain.CurrentDomain.AssemblyResolve += (sender, eventArgs) => + { + if (eventArgs.Name.StartsWith("BenchmarkDotNet, Version=")) + { + var baseDir = Path.GetDirectoryName(assemblyPath); + var path = Path.Combine(baseDir, "BenchmarkDotNet.dll"); + if (File.Exists(path)) + { + return Assembly.LoadFrom(path); + } + } + + // Fallback to default assembly resolver + return null; + }; +#endif + + var assembly = Assembly.LoadFrom(assemblyPath); + + var isDebugAssembly = assembly.IsJitOptimizationDisabled() ?? false; + + return GenericBenchmarksBuilder.GetRunnableBenchmarks(assembly.GetRunnableBenchmarks()) + .Select(type => + { + var benchmarkRunInfo = BenchmarkConverter.TypeToBenchmarks(type); + if (isDebugAssembly) + { + // If the assembly is a debug assembly, then only display them if they will run in-process + // This will allow people to debug their benchmarks using VSTest if they wish. + benchmarkRunInfo = new BenchmarkRunInfo( + benchmarkRunInfo.BenchmarksCases.Where(c => c.GetToolchain().IsInProcess).ToArray(), + benchmarkRunInfo.Type, + benchmarkRunInfo.Config); + } + + return benchmarkRunInfo; + }) + .Where(runInfo => runInfo.BenchmarksCases.Length > 0) + .ToArray(); + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkExecutor.cs b/src/BenchmarkDotNet.TestAdapter/BenchmarkExecutor.cs new file mode 100644 index 0000000000..aad574ed76 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkExecutor.cs @@ -0,0 +1,90 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.TestAdapter.Remoting; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; + +namespace BenchmarkDotNet.TestAdapter +{ + /// + /// A class used for executing benchmarks + /// + internal class BenchmarkExecutor + { + private readonly CancellationTokenSource cts = new(); + + /// + /// Runs all the benchmarks in the given assembly, updating the TestExecutionRecorder as they get run. + /// + /// The dll or exe of the benchmark project. + /// The interface used to record the current test execution progress. + /// + /// An optional list of benchmark IDs specifying which benchmarks to run. + /// These IDs are the same as the ones generated for the VSTest TestCase. + /// + public void RunBenchmarks(string assemblyPath, TestExecutionRecorderWrapper recorder, HashSet? benchmarkIds = null) + { + var benchmarks = BenchmarkEnumerator.GetBenchmarksFromAssemblyPath(assemblyPath); + var testCases = new List(); + + var filteredBenchmarks = new List(); + foreach (var benchmark in benchmarks) + { + var needsJobInfo = benchmark.BenchmarksCases.Select(c => c.Job.DisplayInfo).Distinct().Count() > 1; + var filteredCases = new List(); + foreach (var benchmarkCase in benchmark.BenchmarksCases) + { + var testId = benchmarkCase.GetTestCaseId(); + if (benchmarkIds == null || benchmarkIds.Contains(testId)) + { + filteredCases.Add(benchmarkCase); + testCases.Add(benchmarkCase.ToVsTestCase(assemblyPath, needsJobInfo)); + } + } + + if (filteredCases.Count > 0) + { + filteredBenchmarks.Add(new BenchmarkRunInfo(filteredCases.ToArray(), benchmark.Type, benchmark.Config)); + } + } + + benchmarks = filteredBenchmarks.ToArray(); + + if (benchmarks.Length == 0) + return; + + // Create an event processor which will subscribe to events and push them to VSTest + var eventProcessor = new VsTestEventProcessor(testCases, recorder, cts.Token); + + // Create a logger which will forward all log messages in BDN to the VSTest logger. + var logger = new VsTestLogger(recorder.GetLogger()); + + // Modify all the benchmarks so that the event process and logger is added. + benchmarks = benchmarks + .Select(b => new BenchmarkRunInfo( + b.BenchmarksCases, + b.Type, + b.Config.AddEventProcessor(eventProcessor) + .AddLogger(logger) + .RemoveLoggersOfType() // Console logs are also outputted by VSTestLogger. + .CreateImmutableConfig())) + .ToArray(); + + // Run all the benchmarks, and ensure that any tests that don't have a result yet are sent. + BenchmarkRunner.Run(benchmarks); + eventProcessor.SendUnsentTestResults(); + } + + /// + /// Stop the benchmarks when next able. + /// + public void Cancel() + { + cts.Cancel(); + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkEnumeratorWrapper.cs b/src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkEnumeratorWrapper.cs new file mode 100644 index 0000000000..b3ad68bb23 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkEnumeratorWrapper.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace BenchmarkDotNet.TestAdapter.Remoting +{ + /// + /// A wrapper around the BenchmarkEnumerator for passing data across AppDomain boundaries. + /// + internal class BenchmarkEnumeratorWrapper : MarshalByRefObject + { + /// + /// Gets a list of VSTest TestCases from the given assembly. + /// Each test case is serialized into a string so that it can be used across AppDomain boundaries. + /// + /// The dll or exe of the benchmark project. + /// The serialized test cases. + public List GetTestCasesFromAssemblyPathSerialized(string assemblyPath) + { + var serializedTestCases = new List(); + foreach (var runInfo in BenchmarkEnumerator.GetBenchmarksFromAssemblyPath(assemblyPath)) + { + // If all the benchmarks have the same job, then no need to include job info. + var needsJobInfo = runInfo.BenchmarksCases.Select(c => c.Job.DisplayInfo).Distinct().Count() > 1; + foreach (var benchmarkCase in runInfo.BenchmarksCases) + { + var testCase = benchmarkCase.ToVsTestCase(assemblyPath, needsJobInfo); + serializedTestCases.Add(SerializationHelpers.Serialize(testCase)); + } + } + + return serializedTestCases; + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkExecutorWrapper.cs b/src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkExecutorWrapper.cs new file mode 100644 index 0000000000..28444ae8db --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkExecutorWrapper.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; + +namespace BenchmarkDotNet.TestAdapter.Remoting +{ + /// + /// A wrapper around the BenchmarkExecutor that works across AppDomain boundaries. + /// + internal class BenchmarkExecutorWrapper : MarshalByRefObject + { + private readonly BenchmarkExecutor benchmarkExecutor = new(); + + public void RunBenchmarks(string assemblyPath, TestExecutionRecorderWrapper recorder, HashSet? benchmarkIds = null) + { + benchmarkExecutor.RunBenchmarks(assemblyPath, recorder, benchmarkIds); + } + + public void Cancel() + { + benchmarkExecutor.Cancel(); + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Remoting/MessageLoggerWrapper.cs b/src/BenchmarkDotNet.TestAdapter/Remoting/MessageLoggerWrapper.cs new file mode 100644 index 0000000000..00c4f5325f --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Remoting/MessageLoggerWrapper.cs @@ -0,0 +1,23 @@ +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +using System; + +namespace BenchmarkDotNet.TestAdapter.Remoting +{ + /// + /// A wrapper around an IMessageLogger that works across AppDomain boundaries. + /// + internal class MessageLoggerWrapper : MarshalByRefObject, IMessageLogger + { + private readonly IMessageLogger logger; + + public MessageLoggerWrapper(IMessageLogger logger) + { + this.logger = logger; + } + + public void SendMessage(TestMessageLevel testMessageLevel, string message) + { + logger.SendMessage(testMessageLevel, message); + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Remoting/SerializationHelpers.cs b/src/BenchmarkDotNet.TestAdapter/Remoting/SerializationHelpers.cs new file mode 100644 index 0000000000..5b13bd5175 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Remoting/SerializationHelpers.cs @@ -0,0 +1,26 @@ +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; + +namespace BenchmarkDotNet.TestAdapter.Remoting +{ + /// + /// A set of helper methods for serializing and deserializing the VSTest TestCases and TestReports. + /// + internal static class SerializationHelpers + { + // Version number of the VSTest protocol that the adapter supports. Only needs to be updated when + // the VSTest protocol has a change and this test adapter wishes to take a dependency on it. + // A list of protocol versions and a summary of the changes that were made in them can be found here: + // https://github.com/microsoft/vstest/blob/main/docs/Overview.md#protocolversion-request + private const int VsTestProtocolVersion = 7; + + public static string Serialize(T data) + { + return JsonDataSerializer.Instance.Serialize(data, version: VsTestProtocolVersion); + } + + public static T Deserialize(string data) + { + return JsonDataSerializer.Instance.Deserialize(data, version: VsTestProtocolVersion)!; + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Remoting/TestExecutionRecorderWrapper.cs b/src/BenchmarkDotNet.TestAdapter/Remoting/TestExecutionRecorderWrapper.cs new file mode 100644 index 0000000000..0669e79019 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Remoting/TestExecutionRecorderWrapper.cs @@ -0,0 +1,39 @@ +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; +using System; + +namespace BenchmarkDotNet.TestAdapter.Remoting +{ + /// + /// A wrapper around the ITestExecutionRecorder which works across AppDomain boundaries. + /// + internal class TestExecutionRecorderWrapper : MarshalByRefObject + { + private readonly ITestExecutionRecorder testExecutionRecorder; + + public TestExecutionRecorderWrapper(ITestExecutionRecorder testExecutionRecorder) + { + this.testExecutionRecorder = testExecutionRecorder; + } + + public MessageLoggerWrapper GetLogger() + { + return new MessageLoggerWrapper(testExecutionRecorder); + } + + internal void RecordStart(string serializedTestCase) + { + testExecutionRecorder.RecordStart(SerializationHelpers.Deserialize(serializedTestCase)); + } + + internal void RecordEnd(string serializedTestCase, TestOutcome testOutcome) + { + testExecutionRecorder.RecordEnd(SerializationHelpers.Deserialize(serializedTestCase), testOutcome); + } + + internal void RecordResult(string serializedTestResult) + { + testExecutionRecorder.RecordResult(SerializationHelpers.Deserialize(serializedTestResult)); + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.TestAdapter/Utility/LoggerHelper.cs b/src/BenchmarkDotNet.TestAdapter/Utility/LoggerHelper.cs new file mode 100644 index 0000000000..08238c1927 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Utility/LoggerHelper.cs @@ -0,0 +1,57 @@ +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +using System.Diagnostics; +using System.IO; + +namespace BenchmarkDotNet.TestAdapter; + +internal class LoggerHelper +{ + public LoggerHelper(IMessageLogger logger, Stopwatch stopwatch) + { + InnerLogger = logger; + Stopwatch = stopwatch; + } + + public IMessageLogger InnerLogger { get; private set; } + + public Stopwatch Stopwatch { get; private set; } + + public void Log(string format, params object[] args) + { + SendMessage(TestMessageLevel.Informational, null, string.Format(format, args)); + } + + public void LogWithSource(string source, string format, params object[] args) + { + SendMessage(TestMessageLevel.Informational, source, string.Format(format, args)); + } + + public void LogError(string format, params object[] args) + { + SendMessage(TestMessageLevel.Error, null, string.Format(format, args)); + } + + public void LogErrorWithSource(string source, string format, params object[] args) + { + SendMessage(TestMessageLevel.Error, source, string.Format(format, args)); + } + + public void LogWarning(string format, params object[] args) + { + SendMessage(TestMessageLevel.Warning, null, string.Format(format, args)); + } + + public void LogWarningWithSource(string source, string format, params object[] args) + { + SendMessage(TestMessageLevel.Warning, source, string.Format(format, args)); + } + + private void SendMessage(TestMessageLevel level, string? assemblyName, string message) + { + var assemblyText = assemblyName == null + ? "" : + $"{Path.GetFileNameWithoutExtension(assemblyName)}: "; + + InnerLogger.SendMessage(level, $"[BenchmarkDotNet {Stopwatch.Elapsed:hh\\:mm\\:ss\\.ff}] {assemblyText}{message}"); + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Utility/TestCaseFilter.cs b/src/BenchmarkDotNet.TestAdapter/Utility/TestCaseFilter.cs new file mode 100644 index 0000000000..ffd891bf61 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Utility/TestCaseFilter.cs @@ -0,0 +1,166 @@ +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; + +namespace BenchmarkDotNet.TestAdapter; + +internal class TestCaseFilter +{ + private const string DisplayNameString = "DisplayName"; + private const string FullyQualifiedNameString = "FullyQualifiedName"; + + private readonly HashSet knownTraits; + private List supportedPropertyNames; + private readonly ITestCaseFilterExpression? filterExpression; + private readonly bool successfullyGotFilter; + private readonly bool isDiscovery; + + public TestCaseFilter(IDiscoveryContext discoveryContext, LoggerHelper logger) + { + // Traits are not known at discovery time because we load them from benchmarks + isDiscovery = true; + knownTraits = []; + supportedPropertyNames = GetSupportedPropertyNames(); + successfullyGotFilter = GetTestCaseFilterExpressionFromDiscoveryContext(discoveryContext, logger, out filterExpression); + } + + public TestCaseFilter(IRunContext runContext, LoggerHelper logger, string assemblyFileName, HashSet knownTraits) + { + this.knownTraits = knownTraits; + supportedPropertyNames = GetSupportedPropertyNames(); + successfullyGotFilter = GetTestCaseFilterExpression(runContext, logger, assemblyFileName, out filterExpression); + } + + public string GetTestCaseFilterValue() + { + return successfullyGotFilter + ? filterExpression?.TestCaseFilterValue ?? "" + : ""; + } + + public bool MatchTestCase(TestCase testCase) + { + if (!successfullyGotFilter) + { + // Had an error while getting filter, match no testcase to ensure discovered test list is empty + return false; + } + else if (filterExpression == null) + { + // No filter specified, keep every testcase + return true; + } + + return filterExpression.MatchTestCase(testCase, p => PropertyProvider(testCase, p)); + } + + public object? PropertyProvider(TestCase testCase, string name) + { + // Traits filtering + if (isDiscovery || knownTraits.Contains(name)) + { + var result = new List(); + + foreach (var trait in GetTraits(testCase)) + if (string.Equals(trait.Key, name, StringComparison.OrdinalIgnoreCase)) + result.Add(trait.Value); + + if (result.Count > 0) + return result.ToArray(); + } + + // Property filtering + switch (name.ToLowerInvariant()) + { + // FullyQualifiedName + case "fullyqualifiedname": + return testCase.FullyQualifiedName; + // DisplayName + case "displayname": + return testCase.DisplayName; + default: + return null; + } + } + + private bool GetTestCaseFilterExpression(IRunContext runContext, LoggerHelper logger, string assemblyFileName, out ITestCaseFilterExpression? filter) + { + filter = null; + + try + { + filter = runContext.GetTestCaseFilter(supportedPropertyNames, null!); + return true; + } + catch (TestPlatformFormatException e) + { + logger.LogWarning("{0}: Exception filtering tests: {1}", Path.GetFileNameWithoutExtension(assemblyFileName), e.Message); + return false; + } + } + + private bool GetTestCaseFilterExpressionFromDiscoveryContext(IDiscoveryContext discoveryContext, LoggerHelper logger, out ITestCaseFilterExpression? filter) + { + filter = null; + + if (discoveryContext is IRunContext runContext) + { + try + { + filter = runContext.GetTestCaseFilter(supportedPropertyNames, null!); + return true; + } + catch (TestPlatformException e) + { + logger.LogWarning("Exception filtering tests: {0}", e.Message); + return false; + } + } + else + { + try + { + // GetTestCaseFilter is present on DiscoveryContext but not in IDiscoveryContext interface + var method = discoveryContext.GetType().GetRuntimeMethod("GetTestCaseFilter", [typeof(IEnumerable), typeof(Func)]); + filter = (ITestCaseFilterExpression)method?.Invoke(discoveryContext, [supportedPropertyNames, null])!; + + return true; + } + catch (TargetInvocationException e) + { + if (e?.InnerException is TestPlatformException ex) + { + logger.LogWarning("Exception filtering tests: {0}", ex.InnerException.Message ?? ""); + return false; + } + + throw e!.InnerException; + } + } + } + + private List GetSupportedPropertyNames() + { + // Returns the set of well-known property names usually used with the Test Plugins (Used Test Traits + DisplayName + FullyQualifiedName) + if (supportedPropertyNames == null) + { + supportedPropertyNames = knownTraits.ToList(); + supportedPropertyNames.Add(DisplayNameString); + supportedPropertyNames.Add(FullyQualifiedNameString); + } + + return supportedPropertyNames; + } + + private static IEnumerable> GetTraits(TestCase testCase) + { + var traitProperty = TestProperty.Find("TestObject.Traits"); + return traitProperty != null + ? testCase.GetPropertyValue(traitProperty, Array.Empty>()) + : []; + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs b/src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs new file mode 100644 index 0000000000..3a3e155bf4 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs @@ -0,0 +1,249 @@ +using BenchmarkDotNet.TestAdapter.Remoting; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading; + +namespace BenchmarkDotNet.TestAdapter +{ + /// + /// Discovers and executes benchmarks using the VSTest protocol. + /// + [ExtensionUri(ExecutorUriString)] + [DefaultExecutorUri(ExecutorUriString)] + [FileExtension(".dll")] + [FileExtension(".exe")] + public class VsTestAdapter : ITestExecutor, ITestDiscoverer + { + // This URI is used to identify the adapter. + internal const string ExecutorUriString = "executor://BenchmarkDotNet.TestAdapter"; + internal static readonly Uri ExecutorUri = new Uri(ExecutorUriString); + + /// + /// Cancellation token used to stop any benchmarks that are currently running. + /// + private CancellationTokenSource? cts = null; + + /// + /// Discovers the benchmarks. + /// + /// List of assemblies to search for benchmarks in. + /// A context that the discovery is performed in. + /// Logger that sends messages back to VSTest host. + /// Interface that provides methods for sending discovered benchmarks back to the host. + public void DiscoverTests( + IEnumerable sources, + IDiscoveryContext discoveryContext, + IMessageLogger logger, + ITestCaseDiscoverySink discoverySink) + { + var stopwatch = Stopwatch.StartNew(); + var loggerHelper = new LoggerHelper(logger, stopwatch); + var testCaseFilter = new TestCaseFilter(discoveryContext, loggerHelper); + + foreach (var source in sources) + { + ValidateSourceIsAssemblyOrThrow(source); + foreach (var testCase in GetVsTestCasesFromAssembly(source, logger)) + { + if (!testCaseFilter.MatchTestCase(testCase)) + continue; + + discoverySink.SendTestCase(testCase); + } + } + } + + /// + /// Runs a given set of test cases that represent benchmarks. + /// + /// The tests to run. + /// A context that the run is performed in. + /// Interface used for communicating with the VSTest host. + public void RunTests(IEnumerable? tests, IRunContext? runContext, IFrameworkHandle? frameworkHandle) + { + if (tests == null) + throw new ArgumentNullException(nameof(tests)); + if (frameworkHandle == null) + throw new ArgumentNullException(nameof(frameworkHandle)); + + cts ??= new CancellationTokenSource(); + + var stopwatch = Stopwatch.StartNew(); + var logger = new LoggerHelper(frameworkHandle, stopwatch); + + foreach (var testsPerAssembly in tests.GroupBy(t => t.Source)) + { + RunBenchmarks(testsPerAssembly.Key, frameworkHandle, testsPerAssembly); + } + + cts = null; + } + + /// + /// Runs all/filtered benchmarks in the given set of sources (assemblies). + /// + /// The assemblies to run. + /// A context that the run is performed in. + /// Interface used for communicating with the VSTest host. + public void RunTests(IEnumerable? sources, IRunContext? runContext, IFrameworkHandle? frameworkHandle) + { + if (sources == null) + throw new ArgumentNullException(nameof(sources)); + if (frameworkHandle == null) + throw new ArgumentNullException(nameof(frameworkHandle)); + + cts ??= new CancellationTokenSource(); + + var stopwatch = Stopwatch.StartNew(); + var logger = new LoggerHelper(frameworkHandle, stopwatch); + + foreach (var source in sources) + { + var filter = new TestCaseFilter(runContext!, logger, source, ["Category"]); + if (filter.GetTestCaseFilterValue() != "") + { + var discoveredBenchmarks = GetVsTestCasesFromAssembly(source, frameworkHandle); + var filteredTestCases = discoveredBenchmarks.Where(x => filter.MatchTestCase(x)) + .ToArray(); + + if (filteredTestCases.Length == 0) + continue; + + // Run filtered tests. + RunBenchmarks(source, frameworkHandle, filteredTestCases); + } + else + { + // Run all benchmarks + RunBenchmarks(source, frameworkHandle); + } + } + + + cts = null; + } + + /// + /// Stops any currently running benchmarks. + /// + public void Cancel() + { + cts?.Cancel(); + } + + /// + /// Gets the VSTest test cases in the given assembly. + /// + /// The dll or exe of the benchmark project. + /// A logger that sends logs to VSTest. + /// The VSTest test cases inside the given assembly. + private static List GetVsTestCasesFromAssembly(string assemblyPath, IMessageLogger logger) + { + try + { + // Ensure that the test enumeration is done inside the context of the source directory. + var enumerator = (BenchmarkEnumeratorWrapper)CreateIsolatedType(typeof(BenchmarkEnumeratorWrapper), assemblyPath); + var testCases = enumerator + .GetTestCasesFromAssemblyPathSerialized(assemblyPath) + .Select(SerializationHelpers.Deserialize) + .ToList(); + + // Validate that all test ids are unique + var idLookup = new Dictionary(); + foreach (var testCase in testCases) + { + if (idLookup.TryGetValue(testCase.Id, out var matchingCase)) + throw new Exception($"Encountered Duplicate Test ID: '{testCase.DisplayName}' and '{matchingCase}'"); + + idLookup[testCase.Id] = testCase.DisplayName; + } + + return testCases; + } + catch (Exception ex) + { + logger.SendMessage(TestMessageLevel.Error, $"Failed to load benchmarks from assembly\n{ex}"); + throw; + } + } + + /// + /// Runs the benchmarks in the given source. + /// + /// The dll or exe of the benchmark project. + /// An interface used to communicate with the VSTest host. + /// + /// The specific test cases to be run if specified. + /// If unspecified, runs all the test cases in the source. + /// + private void RunBenchmarks(string source, IFrameworkHandle frameworkHandle, IEnumerable? testCases = null) + { + ValidateSourceIsAssemblyOrThrow(source); + + // Create a HashSet of all the TestCase IDs to be run if specified. + var caseIds = testCases == null ? null : new HashSet(testCases.Select(c => c.Id)); + + try + { + // Ensure that test execution is done inside the context of the source directory. + var executor = (BenchmarkExecutorWrapper)CreateIsolatedType(typeof(BenchmarkExecutorWrapper), source); + cts?.Token.Register(executor.Cancel); + + executor.RunBenchmarks(source, new TestExecutionRecorderWrapper(frameworkHandle), caseIds); + } + catch (Exception ex) + { + frameworkHandle.SendMessage(TestMessageLevel.Error, $"Failed to run benchmarks in assembly\n{ex}"); + throw; + } + } + + /// + /// This will create the given type in a child AppDomain when used in .NET Framework. + /// If not in the .NET Framework, it will use the current AppDomain. + /// + /// The type to create. + /// The dll or exe of the benchmark project. + /// The created object. + private static object CreateIsolatedType(Type type, string assemblyPath) + { + // .NET Framework runs require a custom AppDomain to be set up to run the benchmarks in because otherwise, + // all the assemblies will be loaded from the VSTest console rather than from the directory that the BDN + // program under test lives in. .NET Core assembly resolution is smarter and will correctly load the right + // assembly versions as needed and does not require a custom AppDomain. Unfortunately, the APIs needed to + // create the AppDomain for .NET Framework are not part of .NET Standard, and so a multi-targeting solution + // such as this is required to get this to work. This same approach is also used by other .NET unit testing + // libraries as well, further justifying this approach to solving how to get the correct assemblies loaded. +#if NETFRAMEWORK + var appBase = Path.GetDirectoryName(assemblyPath); + var setup = new AppDomainSetup { ApplicationBase = appBase }; + var domainName = $"Isolated Domain for {type.Name}"; + var appDomain = AppDomain.CreateDomain(domainName, null, setup); + return appDomain.CreateInstanceAndUnwrap( + type.Assembly.FullName, type.FullName, false, BindingFlags.Default, null, null, null, null); +#else + return Activator.CreateInstance(type); +#endif + } + + private static void ValidateSourceIsAssemblyOrThrow(string source) + { + if (string.IsNullOrEmpty(source)) + throw new ArgumentException($"'{nameof(source)}' cannot be null or whitespace.", nameof(source)); + + if (!Path.HasExtension(source)) + throw new NotSupportedException($"Missing extension on source '{source}', must have the extension '.dll' or '.exe'."); + + var extension = Path.GetExtension(source); + if (!string.Equals(extension, ".dll", StringComparison.OrdinalIgnoreCase) && !string.Equals(extension, ".exe", StringComparison.OrdinalIgnoreCase)) + throw new NotSupportedException($"Unsupported extension on source '{source}', must have the extension '.dll' or '.exe'."); + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs b/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs new file mode 100644 index 0000000000..f002e7eb29 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs @@ -0,0 +1,198 @@ +using BenchmarkDotNet.EventProcessors; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.TestAdapter.Remoting; +using BenchmarkDotNet.Toolchains.Results; +using BenchmarkDotNet.Validators; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Perfolizer.Mathematics.Histograms; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Threading; + +namespace BenchmarkDotNet.TestAdapter +{ + /// + /// An event processor which will pass on benchmark execution information to VSTest. + /// + internal class VsTestEventProcessor : EventProcessor + { + private readonly Dictionary cases; + private readonly TestExecutionRecorderWrapper recorder; + private readonly CancellationToken cancellationToken; + private readonly Stopwatch runTimerStopwatch = new(); + private readonly Dictionary testResults = new(); + private readonly HashSet sentTestResults = new(); + + public VsTestEventProcessor( + List cases, + TestExecutionRecorderWrapper recorder, + CancellationToken cancellationToken) + { + this.cases = cases.ToDictionary(c => c.Id); + this.recorder = recorder; + this.cancellationToken = cancellationToken; + } + + public override void OnValidationError(ValidationError validationError) + { + // If the error is not linked to a benchmark case, then set the error on all benchmarks + var errorCases = validationError.BenchmarkCase == null + ? cases.Values.ToList() + : new List { cases[validationError.BenchmarkCase.GetTestCaseId()] }; + foreach (var testCase in errorCases) + { + var testResult = GetOrCreateTestResult(testCase); + + if (validationError.IsCritical) + { + // Fail if there is a critical validation error + testResult.Outcome = TestOutcome.Failed; + + // Append validation error message to end of test case error message + testResult.ErrorMessage = testResult.ErrorMessage == null + ? validationError.Message + : $"{testResult.ErrorMessage}\n{validationError.Message}"; + + // The test result is not sent yet, in case there are multiple validation errors that need to be sent. + } + else + { + // If the validation error is not critical, append it as a message + testResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, $"WARNING: {validationError.Message}\n")); + } + } + } + + public override void OnBuildComplete(BuildPartition buildPartition, BuildResult buildResult) + { + // Only need to handle build failures + if (!buildResult.IsBuildSuccess) + { + foreach (var benchmarkBuildInfo in buildPartition.Benchmarks) + { + var testCase = cases[benchmarkBuildInfo.BenchmarkCase.GetTestCaseId()]; + var testResult = GetOrCreateTestResult(testCase); + + if (buildResult.GenerateException != null) + testResult.ErrorMessage = $"// Generate Exception: {buildResult.GenerateException.Message}"; + else if (!buildResult.IsBuildSuccess && buildResult.TryToExplainFailureReason(out string reason)) + testResult.ErrorMessage = $"// Build Error: {reason}"; + else if (buildResult.ErrorMessage != null) + testResult.ErrorMessage = $"// Build Error: {buildResult.ErrorMessage}"; + testResult.Outcome = TestOutcome.Failed; + + // Send the result immediately + RecordStart(testCase); + RecordEnd(testCase, testResult.Outcome); + RecordResult(testResult); + sentTestResults.Add(testCase.Id); + } + } + } + + public override void OnStartRunBenchmark(BenchmarkCase benchmarkCase) + { + // TODO: add proper cancellation support to BDN so that we don't need to do cancellation through the event processor + cancellationToken.ThrowIfCancellationRequested(); + + var testCase = cases[benchmarkCase.GetTestCaseId()]; + var testResult = GetOrCreateTestResult(testCase); + testResult.StartTime = DateTimeOffset.UtcNow; + + RecordStart(testCase); + runTimerStopwatch.Restart(); + } + + public override void OnEndRunBenchmark(BenchmarkCase benchmarkCase, BenchmarkReport report) + { + var testCase = cases[benchmarkCase.GetTestCaseId()]; + var testResult = GetOrCreateTestResult(testCase); + testResult.EndTime = DateTimeOffset.UtcNow; + testResult.Duration = runTimerStopwatch.Elapsed; + testResult.Outcome = report.Success ? TestOutcome.Passed : TestOutcome.Failed; + + var resultRuns = report.GetResultRuns(); + + // Provide the raw result runs data. + testResult.SetPropertyValue(VsTestProperties.Measurement, resultRuns.Select(m => m.Nanoseconds.ToString()).ToArray()); + + // Add a message to the TestResult which contains the results summary. + testResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, report.BenchmarkCase.DisplayInfo + "\n")); + testResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, $"Runtime = {report.GetRuntimeInfo()}; GC = {report.GetGcInfo()}\n")); + + var statistics = resultRuns.GetStatistics(); + var cultureInfo = CultureInfo.InvariantCulture; + var formatter = statistics.CreateNanosecondFormatter(cultureInfo); + + var builder = new StringBuilder(); + var histogram = HistogramBuilder.Adaptive.Build(statistics.Sample.Values); + builder.AppendLine("-------------------- Histogram --------------------"); + builder.AppendLine(histogram.ToString(formatter)); + builder.AppendLine("---------------------------------------------------"); + + var statisticsOutput = statistics.ToString(cultureInfo, formatter, calcHistogram: false); + builder.AppendLine(statisticsOutput); + + testResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, builder.ToString())); + + RecordEnd(testResult.TestCase, testResult.Outcome); + RecordResult(testResult); + sentTestResults.Add(testCase.Id); + } + + /// + /// Iterate through all the benchmarks that were scheduled to run, and if they haven't been sent yet, send the result through. + /// + public void SendUnsentTestResults() + { + foreach (var testCase in cases.Values) + { + if (!sentTestResults.Contains(testCase.Id)) + { + var testResult = GetOrCreateTestResult(testCase); + if (testResult.Outcome == TestOutcome.None) + testResult.Outcome = TestOutcome.Skipped; + RecordStart(testCase); + RecordEnd(testCase, testResult.Outcome); + RecordResult(testResult); + } + } + } + + private TestResult GetOrCreateTestResult(TestCase testCase) + { + if (testResults.TryGetValue(testCase.Id, out var testResult)) + return testResult; + + var newResult = new TestResult(testCase) + { + ComputerName = Environment.MachineName, + DisplayName = testCase.DisplayName + }; + + testResults[testCase.Id] = newResult; + return newResult; + } + + private void RecordStart(TestCase testCase) + { + recorder.RecordStart(SerializationHelpers.Serialize(testCase)); + } + + private void RecordEnd(TestCase testCase, TestOutcome testOutcome) + { + recorder.RecordEnd(SerializationHelpers.Serialize(testCase), testOutcome); + } + + private void RecordResult(TestResult testResult) + { + recorder.RecordResult(SerializationHelpers.Serialize(testResult)); + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/VSTestLogger.cs b/src/BenchmarkDotNet.TestAdapter/VSTestLogger.cs new file mode 100644 index 0000000000..c9a8de620f --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/VSTestLogger.cs @@ -0,0 +1,62 @@ +using BenchmarkDotNet.Loggers; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +using System.Text; + +namespace BenchmarkDotNet.TestAdapter +{ + /// + /// A class to send logs from BDN to the VSTest output log. + /// + internal sealed class VsTestLogger : ILogger + { + private readonly IMessageLogger messageLogger; + private readonly StringBuilder currentLine = new StringBuilder(); + private TestMessageLevel currentLevel = TestMessageLevel.Informational; + + public VsTestLogger(IMessageLogger logger) + { + messageLogger = logger; + } + + public string Id => nameof(VsTestLogger); + + public int Priority => 0; + + public void Flush() + { + WriteLine(); + } + + public void Write(LogKind logKind, string text) + { + currentLine.Append(text); + + // Assume that if the log kind is an error, that the whole line is treated as an error + // The level will be reset to Informational when WriteLine() is called. + currentLevel = logKind switch + { + LogKind.Error => TestMessageLevel.Error, + LogKind.Warning => TestMessageLevel.Warning, + _ => currentLevel + }; + } + + public void WriteLine() + { + // The VSTest logger throws an error on logging empty or whitespace strings, so skip them. + if (currentLine.Length == 0) + return; + + messageLogger.SendMessage(currentLevel, currentLine.ToString()); + + currentLevel = TestMessageLevel.Informational; + currentLine.Clear(); + } + + public void WriteLine(LogKind logKind, string text) + { + Write(logKind, text); + WriteLine(); + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/VSTestProperties.cs b/src/BenchmarkDotNet.TestAdapter/VSTestProperties.cs new file mode 100644 index 0000000000..6bcbdcf299 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/VSTestProperties.cs @@ -0,0 +1,22 @@ +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +namespace BenchmarkDotNet.TestAdapter +{ + /// + /// A class that contains all the custom properties that can be set on VSTest TestCase and TestResults. + /// Some of these properties are well known as they are also used by VSTest adapters for other test libraries. + /// + internal static class VsTestProperties + { + /// + /// A test property used for storing the test results so that they could be accessed + /// programmatically from a custom VSTest runner. + /// + internal static readonly TestProperty Measurement = TestProperty.Register( + "BenchmarkDotNet.TestAdapter.Measurements", + "Measurements", + typeof(string[]), + TestPropertyAttributes.Hidden, + typeof(TestResult)); + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props b/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props new file mode 100644 index 0000000000..7184842aba --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props @@ -0,0 +1,23 @@ + + + $(MSBuildThisFileDirectory)..\entrypoints\ + + false + + + + + + + + + + + false + + + diff --git a/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.cs b/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.cs new file mode 100644 index 0000000000..4f9036a00c --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.cs @@ -0,0 +1,11 @@ +// +// Exclude this file from StyleCop analysis. This file isn't generated but is added to projects. +// + +using BenchmarkDotNet.Running; +using System.Reflection; + +public class __AutoGeneratedEntryPointClass +{ + public static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(__AutoGeneratedEntryPointClass).Assembly).Run(args); +} diff --git a/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.fs b/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.fs new file mode 100644 index 0000000000..7e305320b8 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.fs @@ -0,0 +1,13 @@ +// +// Exclude this file from StyleCop analysis. This file isn't generated but is added to projects. +// + +module __AutoGeneratedEntryPointClass +open System.Reflection; +open BenchmarkDotNet.Running + +type internal __Marker = interface end // Used to help locale current assembly +[] +let main argv = + BenchmarkSwitcher.FromAssembly(typeof<__Marker>.Assembly).Run(argv) |> ignore + 0 // return an integer exit code diff --git a/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.vb b/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.vb new file mode 100644 index 0000000000..24458cafca --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.vb @@ -0,0 +1,14 @@ +REM +REM Exclude this file from StyleCop analysis. This file isn't generated but is added to projects. +REM + +Imports System.Reflection +Imports BenchmarkDotNet.Running + +Namespace Global + Module __AutoGeneratedEntryPointClass + Sub Main(args As String()) + Dim summary = BenchmarkSwitcher.FromAssembly(MethodBase.GetCurrentMethod().Module.Assembly).Run(args) + End Sub + End Module +End Namespace \ No newline at end of file diff --git a/src/BenchmarkDotNet/Analysers/BaselineCustomAnalyzer.cs b/src/BenchmarkDotNet/Analysers/BaselineCustomAnalyzer.cs index 51663e4085..9f2ff0028c 100644 --- a/src/BenchmarkDotNet/Analysers/BaselineCustomAnalyzer.cs +++ b/src/BenchmarkDotNet/Analysers/BaselineCustomAnalyzer.cs @@ -25,13 +25,14 @@ protected override IEnumerable AnalyseSummary(Summary summary) foreach (var benchmarkCase in summary.BenchmarksCases) { - string logicalGroupKey = summary.GetLogicalGroupKey(benchmarkCase); + string? logicalGroupKey = summary.GetLogicalGroupKey(benchmarkCase); var baseline = summary.GetBaseline(logicalGroupKey); if (BaselineCustomColumn.ResultsAreInvalid(summary, benchmarkCase, baseline) == false) continue; var message = "A question mark '?' symbol indicates that it was not possible to compute the " + - $"({columnNames}) column(s) because the baseline value is too close to zero."; + $"({columnNames}) column(s) because the baseline or benchmark could not be found, or " + + $"the baseline value is too close to zero."; yield return Conclusion.CreateWarning(Id, message); } diff --git a/src/BenchmarkDotNet/Analysers/ConclusionHelper.cs b/src/BenchmarkDotNet/Analysers/ConclusionHelper.cs index 9542230606..af872414f2 100644 --- a/src/BenchmarkDotNet/Analysers/ConclusionHelper.cs +++ b/src/BenchmarkDotNet/Analysers/ConclusionHelper.cs @@ -10,7 +10,7 @@ public static class ConclusionHelper public static void Print(ILogger logger, IEnumerable conclusions) { PrintFiltered(conclusions, ConclusionKind.Error, "Errors", logger.WriteLineError); - PrintFiltered(conclusions, ConclusionKind.Warning, "Warnings", logger.WriteLineError); + PrintFiltered(conclusions, ConclusionKind.Warning, "Warnings", logger.WriteLineWarning); PrintFiltered(conclusions, ConclusionKind.Hint, "Hints", logger.WriteLineHint); } diff --git a/src/BenchmarkDotNet/Analysers/MultimodalDistributionAnalyzer.cs b/src/BenchmarkDotNet/Analysers/MultimodalDistributionAnalyzer.cs index 151ac70be2..e1e4465ef6 100644 --- a/src/BenchmarkDotNet/Analysers/MultimodalDistributionAnalyzer.cs +++ b/src/BenchmarkDotNet/Analysers/MultimodalDistributionAnalyzer.cs @@ -20,7 +20,7 @@ protected override IEnumerable AnalyseReport(BenchmarkReport report, if (statistics == null || statistics.N < EngineResolver.DefaultMinWorkloadIterationCount) yield break; - double mValue = MValueCalculator.Calculate(statistics.OriginalValues); + double mValue = MValueCalculator.Calculate(statistics.Sample.Values); if (mValue > 4.2) yield return Create("is multimodal", mValue, report, summary.GetCultureInfo()); else if (mValue > 3.2) diff --git a/src/BenchmarkDotNet/Analysers/OutliersAnalyser.cs b/src/BenchmarkDotNet/Analysers/OutliersAnalyser.cs index c5c54f8c91..570ea7806e 100644 --- a/src/BenchmarkDotNet/Analysers/OutliersAnalyser.cs +++ b/src/BenchmarkDotNet/Analysers/OutliersAnalyser.cs @@ -3,6 +3,7 @@ using System.Linq; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; using JetBrains.Annotations; @@ -53,7 +54,7 @@ string Format(int n, string verb) return $"{n} {words} {verb}"; } - var rangeMessages = new List { GetRangeMessage(lowerOutliers, cultureInfo), GetRangeMessage(upperOutliers, cultureInfo) }; + var rangeMessages = new List { GetRangeMessage(lowerOutliers), GetRangeMessage(upperOutliers) }; rangeMessages.RemoveAll(string.IsNullOrEmpty); string rangeMessage = rangeMessages.Any() ? " (" + string.Join(", ", rangeMessages) + ")" @@ -66,20 +67,17 @@ string Format(int n, string verb) return Format(actualOutliers.Length, "removed") + ", " + Format(allOutliers.Length, "detected") + rangeMessage; } - private static string? GetRangeMessage(double[] values, CultureInfo cultureInfo) + private static string? GetRangeMessage(double[] values) { - string Format(double value) => TimeInterval.FromNanoseconds(value).ToString(cultureInfo, "N2"); + string Format(double value) => TimeInterval.FromNanoseconds(value).ToDefaultString("N2"); - switch (values.Length) { - case 0: - return null; - case 1: - return Format(values.First()); - case 2: - return Format(values.Min()) + ", " + Format(values.Max()); - default: - return Format(values.Min()) + ".." + Format(values.Max()); - } + return values.Length switch + { + 0 => null, + 1 => Format(values.First()), + 2 => Format(values.Min()) + ", " + Format(values.Max()), + _ => Format(values.Min()) + ".." + Format(values.Max()) + }; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Analysers/ZeroMeasurementAnalyser.cs b/src/BenchmarkDotNet/Analysers/ZeroMeasurementAnalyser.cs index 240f2e8475..4d83a5d154 100644 --- a/src/BenchmarkDotNet/Analysers/ZeroMeasurementAnalyser.cs +++ b/src/BenchmarkDotNet/Analysers/ZeroMeasurementAnalyser.cs @@ -19,7 +19,7 @@ private ZeroMeasurementAnalyser() { } protected override IEnumerable AnalyseReport(BenchmarkReport report, Summary summary) { - var currentFrequency = summary.HostEnvironmentInfo.CpuInfo.Value.MaxFrequency; + var currentFrequency = summary.HostEnvironmentInfo.Cpu.Value.MaxFrequency(); if (!currentFrequency.HasValue || currentFrequency <= 0) currentFrequency = FallbackCpuResolutionValue.ToFrequency(); @@ -28,17 +28,17 @@ protected override IEnumerable AnalyseReport(BenchmarkReport report, var workloadMeasurements = entire.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).ToArray(); if (workloadMeasurements.IsEmpty()) yield break; - var workload = workloadMeasurements.GetStatistics(); + var workloadSample = workloadMeasurements.GetStatistics().Sample; var threshold = currentFrequency.Value.ToResolution().Nanoseconds / 2; var zeroMeasurement = overheadMeasurements.Any() - ? ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workload.WithoutOutliers(), overheadMeasurements.GetStatistics().WithoutOutliers()) - : ZeroMeasurementHelper.CheckZeroMeasurementOneSample(workload.WithoutOutliers(), threshold); + ? ZeroMeasurementHelper.AreIndistinguishable(workloadSample, overheadMeasurements.GetStatistics().Sample) + : ZeroMeasurementHelper.IsNegligible(workloadSample, threshold); if (zeroMeasurement) yield return CreateWarning("The method duration is indistinguishable from the empty method duration", - report, false); + report, false); } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs index 5e3f650167..9079bc54d9 100644 --- a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs +++ b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs @@ -1,30 +1,39 @@ +using BenchmarkDotNet.Mathematics; +using Perfolizer; +using Perfolizer.Horology; +using Perfolizer.Mathematics.Common; +using Perfolizer.Mathematics.GenericEstimators; using Perfolizer.Mathematics.SignificanceTesting; -using Perfolizer.Mathematics.Thresholds; +using Perfolizer.Mathematics.SignificanceTesting.MannWhitney; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Analysers { - public static class ZeroMeasurementHelper + internal static class ZeroMeasurementHelper { - /// - /// Checks distribution against Zero Measurement hypothesis in case of known threshold - /// - /// True if measurement is ZeroMeasurement - public static bool CheckZeroMeasurementOneSample(double[] results, double threshold) + public static bool IsNegligible(Sample results, double threshold) => HodgesLehmannEstimator.Instance.Median(results) < threshold; + public static bool IsNoticeable(Sample results, double threshold) => !IsNegligible(results, threshold); + + public static bool AreIndistinguishable(double[] workload, double[] overhead, Threshold? threshold = null) { - if (results.Length < 3) - return false; - return !StudentTest.Instance.IsGreater(results, threshold).NullHypothesisIsRejected; + var workloadSample = new Sample(workload, TimeUnit.Nanosecond); + var overheadSample = new Sample(overhead, TimeUnit.Nanosecond); + return AreIndistinguishable(workloadSample, overheadSample, threshold); } - /// - /// Checks distribution against Zero Measurement hypothesis in case of two samples - /// - /// True if measurement is ZeroMeasurement - public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead, Threshold? threshold = null) + public static bool AreIndistinguishable(Sample workload, Sample overhead, Threshold? threshold = null) { - if (workload.Length < 3 || overhead.Length < 3) + threshold ??= MathHelper.DefaultThreshold; + var tost = new SimpleEquivalenceTest(MannWhitneyTest.Instance); + if (workload.Size == 1 || overhead.Size == 1) return false; - return !WelchTest.Instance.IsGreater(workload, overhead, threshold).NullHypothesisIsRejected; + return tost.Perform(workload, overhead, threshold, SignificanceLevel.P1E5) == ComparisonResult.Indistinguishable; } + + public static bool AreDistinguishable(double[] workload, double[] overhead, Threshold? threshold = null) => + !AreIndistinguishable(workload, overhead, threshold); + + public static bool AreDistinguishable(Sample workload, Sample overhead, Threshold? threshold = null) => + !AreIndistinguishable(workload, overhead, threshold); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/Columns/ConfidenceIntervalErrorColumnAttribute.cs b/src/BenchmarkDotNet/Attributes/Columns/ConfidenceIntervalErrorColumnAttribute.cs index d0d1f90faf..b1b87d5241 100644 --- a/src/BenchmarkDotNet/Attributes/Columns/ConfidenceIntervalErrorColumnAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Columns/ConfidenceIntervalErrorColumnAttribute.cs @@ -7,8 +7,7 @@ namespace BenchmarkDotNet.Attributes [PublicAPI] public class ConfidenceIntervalErrorColumnAttribute : ColumnConfigBaseAttribute { - public ConfidenceIntervalErrorColumnAttribute(ConfidenceLevel level = ConfidenceLevel.L999) : base(StatisticColumn.CiError(level)) - { - } + public ConfidenceIntervalErrorColumnAttribute() : base(StatisticColumn.CiError(ConfidenceLevel.L999)) { } + public ConfidenceIntervalErrorColumnAttribute(ConfidenceLevel level) : base(StatisticColumn.CiError(level)) { } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/Columns/OperationsPerSecondAttribute.cs b/src/BenchmarkDotNet/Attributes/Columns/OperationsPerSecondAttribute.cs new file mode 100644 index 0000000000..6315dc3694 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/Columns/OperationsPerSecondAttribute.cs @@ -0,0 +1,10 @@ +using BenchmarkDotNet.Columns; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Attributes +{ + public class OperationsPerSecondAttribute : ColumnConfigBaseAttribute + { + public OperationsPerSecondAttribute() : base(StatisticColumn.OperationsPerSecond) { } + } +} diff --git a/src/BenchmarkDotNet/Attributes/Columns/StdErrorColumnAttribute.cs b/src/BenchmarkDotNet/Attributes/Columns/StdErrorColumnAttribute.cs index 27a22b1e1a..23327e187e 100644 --- a/src/BenchmarkDotNet/Attributes/Columns/StdErrorColumnAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Columns/StdErrorColumnAttribute.cs @@ -4,10 +4,5 @@ namespace BenchmarkDotNet.Attributes { [PublicAPI] - public class StdErrorColumnAttribute : ColumnConfigBaseAttribute - { - public StdErrorColumnAttribute() : base(StatisticColumn.StdErr) - { - } - } + public class StdErrorColumnAttribute() : ColumnConfigBaseAttribute(StatisticColumn.StdErr) { } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/Columns/WelchTTestPValueColumnAttribute.cs b/src/BenchmarkDotNet/Attributes/Columns/WelchTTestPValueColumnAttribute.cs index 797ab57a9d..607f2c4fe2 100644 --- a/src/BenchmarkDotNet/Attributes/Columns/WelchTTestPValueColumnAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Columns/WelchTTestPValueColumnAttribute.cs @@ -1,8 +1,7 @@ using System; using BenchmarkDotNet.Columns; using JetBrains.Annotations; -using Perfolizer.Mathematics.SignificanceTesting; -using Perfolizer.Mathematics.Thresholds; +using Perfolizer.Mathematics.Common; namespace BenchmarkDotNet.Attributes { @@ -10,17 +9,11 @@ namespace BenchmarkDotNet.Attributes [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)] public class StatisticalTestColumnAttribute : ColumnConfigBaseAttribute { - public StatisticalTestColumnAttribute(StatisticalTestKind testKind, ThresholdUnit thresholdUnit, double value, bool showPValues = false) - : base(StatisticalTestColumn.Create(testKind, Threshold.Create(thresholdUnit, value), showPValues)) { } + public StatisticalTestColumnAttribute() : base(StatisticalTestColumn.Create("10%", null)) { } - public StatisticalTestColumnAttribute(StatisticalTestKind testKind, bool showPValues = false) : this(testKind, ThresholdUnit.Ratio, 0.1, showPValues) { } + public StatisticalTestColumnAttribute(string threshold) : base(StatisticalTestColumn.Create(threshold, null)) { } - public StatisticalTestColumnAttribute(bool showPValues = false) : this(StatisticalTestKind.MannWhitney, showPValues) {} - } - - [Obsolete("Use StatisticalTestAttribute")] - public class WelchTTestPValueColumnAttribute : StatisticalTestColumnAttribute - { - public WelchTTestPValueColumnAttribute() : base(StatisticalTestKind.Welch) { } + public StatisticalTestColumnAttribute(string threshold, SignificanceLevel significanceLevel) + : base(StatisticalTestColumn.Create(threshold, significanceLevel)) { } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs b/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs index bff956e968..21f2903124 100644 --- a/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs @@ -9,6 +9,10 @@ public class ExceptionDiagnoserAttribute : Attribute, IConfigSource { public IConfig Config { get; } - public ExceptionDiagnoserAttribute() => Config = ManualConfig.CreateEmpty().AddDiagnoser(ExceptionDiagnoser.Default); + /// Display Exceptions column. True by default. + public ExceptionDiagnoserAttribute(bool displayExceptionsIfZeroValue = true) + { + Config = ManualConfig.CreateEmpty().AddDiagnoser(new ExceptionDiagnoser(new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue))); + } } } diff --git a/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserConfig.cs b/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserConfig.cs new file mode 100644 index 0000000000..86f8d99be4 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserConfig.cs @@ -0,0 +1,20 @@ +using JetBrains.Annotations; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BenchmarkDotNet.Attributes +{ + public class ExceptionDiagnoserConfig + { + /// Determines whether the Exceptions column is displayed when its value is not calculated. True by default. + + [PublicAPI] + public ExceptionDiagnoserConfig(bool displayExceptionsIfZeroValue = true) + { + DisplayExceptionsIfZeroValue = displayExceptionsIfZeroValue; + } + + public bool DisplayExceptionsIfZeroValue { get; } + } +} diff --git a/src/BenchmarkDotNet/Attributes/Exporters/PerfonarExporterAttribute.cs b/src/BenchmarkDotNet/Attributes/Exporters/PerfonarExporterAttribute.cs new file mode 100644 index 0000000000..e7547f3e7a --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/Exporters/PerfonarExporterAttribute.cs @@ -0,0 +1,10 @@ +using System; +using BenchmarkDotNet.Exporters; + +namespace BenchmarkDotNet.Attributes; + +/// +/// IMPORTANT: Not fully implemented yet +/// +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)] +internal class PerfonarExporterAttribute() : ExporterConfigBaseAttribute(new PerfonarJsonExporter(), new PerfonarMdExporter()); \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/OrdererAttribute.cs b/src/BenchmarkDotNet/Attributes/OrdererAttribute.cs index 83b5051895..b20a9108ca 100644 --- a/src/BenchmarkDotNet/Attributes/OrdererAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/OrdererAttribute.cs @@ -9,9 +9,10 @@ public class OrdererAttribute : Attribute, IConfigSource { public OrdererAttribute( SummaryOrderPolicy summaryOrderPolicy = SummaryOrderPolicy.Default, - MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared) + MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared, + JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default) { - Config = ManualConfig.CreateEmpty().WithOrderer(new DefaultOrderer(summaryOrderPolicy, methodOrderPolicy)); + Config = ManualConfig.CreateEmpty().WithOrderer(new DefaultOrderer(summaryOrderPolicy, methodOrderPolicy, jobOrderPolicy)); } public IConfig Config { get; } diff --git a/src/BenchmarkDotNet/Attributes/ThreadingDiagnoserAttribute.cs b/src/BenchmarkDotNet/Attributes/ThreadingDiagnoserAttribute.cs index 7627170b8b..4ad7651bfc 100644 --- a/src/BenchmarkDotNet/Attributes/ThreadingDiagnoserAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/ThreadingDiagnoserAttribute.cs @@ -1,6 +1,7 @@ using System; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; +using JetBrains.Annotations; namespace BenchmarkDotNet.Attributes { @@ -9,6 +10,15 @@ public class ThreadingDiagnoserAttribute : Attribute, IConfigSource { public IConfig Config { get; } - public ThreadingDiagnoserAttribute() => Config = ManualConfig.CreateEmpty().AddDiagnoser(ThreadingDiagnoser.Default); + //public ThreadingDiagnoserAttribute() => Config = ManualConfig.CreateEmpty().AddDiagnoser(ThreadingDiagnoser.Default); + + /// Display configuration for 'LockContentionCount' when it is empty. True (displayed) by default. + /// Display configuration for 'CompletedWorkItemCount' when it is empty. True (displayed) by default. + + [PublicAPI] + public ThreadingDiagnoserAttribute(bool displayLockContentionWhenZero = true, bool displayCompletedWorkItemCountWhenZero = true) + { + Config = ManualConfig.CreateEmpty().AddDiagnoser(new ThreadingDiagnoser(new ThreadingDiagnoserConfig(displayLockContentionWhenZero, displayCompletedWorkItemCountWhenZero))); + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/WakeLockAttribute.cs b/src/BenchmarkDotNet/Attributes/WakeLockAttribute.cs new file mode 100644 index 0000000000..1243ac8f8f --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/WakeLockAttribute.cs @@ -0,0 +1,18 @@ +using BenchmarkDotNet.Configs; +using System; + +namespace BenchmarkDotNet.Attributes +{ + /// + /// Placing a on your assembly or class controls whether the + /// Windows system enters sleep or turns off the display while benchmarks run. + /// + [AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class)] + public sealed class WakeLockAttribute : Attribute, IConfigSource + { + public WakeLockAttribute(WakeLockType wakeLockType) => + Config = ManualConfig.CreateEmpty().WithWakeLock(wakeLockType); + + public IConfig Config { get; } + } +} diff --git a/src/BenchmarkDotNet/BenchmarkDotNet.csproj b/src/BenchmarkDotNet/BenchmarkDotNet.csproj index 50e6733592..7953a036ae 100644 --- a/src/BenchmarkDotNet/BenchmarkDotNet.csproj +++ b/src/BenchmarkDotNet/BenchmarkDotNet.csproj @@ -2,7 +2,7 @@ BenchmarkDotNet - netstandard2.0;net6.0 + netstandard2.0;net6.0;net8.0 true $(NoWarn);1701;1702;1705;1591;3005;NU1702;CS3001;CS3003 BenchmarkDotNet @@ -13,26 +13,25 @@ - - - - - + + + + - - - + + + - + diff --git a/src/BenchmarkDotNet/Characteristics/CharacteristicObject.cs b/src/BenchmarkDotNet/Characteristics/CharacteristicObject.cs index a6d7b37b81..6cb4a983c3 100644 --- a/src/BenchmarkDotNet/Characteristics/CharacteristicObject.cs +++ b/src/BenchmarkDotNet/Characteristics/CharacteristicObject.cs @@ -402,6 +402,12 @@ protected CharacteristicObject UnfreezeCopyCore() var newRoot = (CharacteristicObject)Activator.CreateInstance(GetType()); newRoot.ApplyCore(this); + // Preserve the IdCharacteristic of the original object + if (this.HasValue(IdCharacteristic)) + { + newRoot.SetValue(IdCharacteristic, this.GetValue(IdCharacteristic)); + } + return newRoot; } #endregion diff --git a/src/BenchmarkDotNet/Code/CodeGenerator.cs b/src/BenchmarkDotNet/Code/CodeGenerator.cs index 005564b77c..9a6228dd88 100644 --- a/src/BenchmarkDotNet/Code/CodeGenerator.cs +++ b/src/BenchmarkDotNet/Code/CodeGenerator.cs @@ -56,6 +56,7 @@ internal static string Generate(BuildPartition buildPartition) .Replace("$OverheadImplementation$", provider.OverheadImplementation) .Replace("$ConsumeField$", provider.ConsumeField) .Replace("$JobSetDefinition$", GetJobsSetDefinition(benchmark)) + .Replace("$ParamsInitializer$", GetParamsInitializer(benchmark)) .Replace("$ParamsContent$", GetParamsContent(benchmark)) .Replace("$ArgumentsDefinition$", GetArgumentsDefinition(benchmark)) .Replace("$DeclareArgumentFields$", GetDeclareArgumentFields(benchmark)) @@ -186,7 +187,15 @@ private static DeclarationsProvider GetDeclarationsProvider(Descriptor descripto return new NonVoidDeclarationsProvider(descriptor); } + private static string GetParamsInitializer(BenchmarkCase benchmarkCase) + => string.Join( + ", ", + benchmarkCase.Parameters.Items + .Where(parameter => !parameter.IsArgument && !parameter.IsStatic) + .Select(parameter => $"{parameter.Name} = default")); + // internal for tests + internal static string GetParamsContent(BenchmarkCase benchmarkCase) => string.Join( string.Empty, diff --git a/src/BenchmarkDotNet/Code/DeclarationsProvider.cs b/src/BenchmarkDotNet/Code/DeclarationsProvider.cs index ddf78eb572..7528e8ed62 100644 --- a/src/BenchmarkDotNet/Code/DeclarationsProvider.cs +++ b/src/BenchmarkDotNet/Code/DeclarationsProvider.cs @@ -63,7 +63,7 @@ private string GetMethodName(MethodInfo method) (method.ReturnType.GetGenericTypeDefinition() == typeof(Task<>) || method.ReturnType.GetGenericTypeDefinition() == typeof(ValueTask<>)))) { - return $"() => {method.Name}().GetAwaiter().GetResult()"; + return $"() => BenchmarkDotNet.Helpers.AwaitHelper.GetResult({method.Name}())"; } return method.Name; @@ -149,12 +149,10 @@ internal class TaskDeclarationsProvider : VoidDeclarationsProvider { public TaskDeclarationsProvider(Descriptor descriptor) : base(descriptor) { } - // we use GetAwaiter().GetResult() because it's fastest way to obtain the result in blocking way, - // and will eventually throw actual exception, not aggregated one public override string WorkloadMethodDelegate(string passArguments) - => $"({passArguments}) => {{ {Descriptor.WorkloadMethod.Name}({passArguments}).GetAwaiter().GetResult(); }}"; + => $"({passArguments}) => {{ BenchmarkDotNet.Helpers.AwaitHelper.GetResult({Descriptor.WorkloadMethod.Name}({passArguments})); }}"; - public override string GetWorkloadMethodCall(string passArguments) => $"{Descriptor.WorkloadMethod.Name}({passArguments}).GetAwaiter().GetResult()"; + public override string GetWorkloadMethodCall(string passArguments) => $"BenchmarkDotNet.Helpers.AwaitHelper.GetResult({Descriptor.WorkloadMethod.Name}({passArguments}))"; protected override Type WorkloadMethodReturnType => typeof(void); } @@ -168,11 +166,9 @@ public GenericTaskDeclarationsProvider(Descriptor descriptor) : base(descriptor) protected override Type WorkloadMethodReturnType => Descriptor.WorkloadMethod.ReturnType.GetTypeInfo().GetGenericArguments().Single(); - // we use GetAwaiter().GetResult() because it's fastest way to obtain the result in blocking way, - // and will eventually throw actual exception, not aggregated one public override string WorkloadMethodDelegate(string passArguments) - => $"({passArguments}) => {{ return {Descriptor.WorkloadMethod.Name}({passArguments}).GetAwaiter().GetResult(); }}"; + => $"({passArguments}) => {{ return BenchmarkDotNet.Helpers.AwaitHelper.GetResult({Descriptor.WorkloadMethod.Name}({passArguments})); }}"; - public override string GetWorkloadMethodCall(string passArguments) => $"{Descriptor.WorkloadMethod.Name}({passArguments}).GetAwaiter().GetResult()"; + public override string GetWorkloadMethodCall(string passArguments) => $"BenchmarkDotNet.Helpers.AwaitHelper.GetResult({Descriptor.WorkloadMethod.Name}({passArguments}))"; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Columns/BaselineCustomColumn.cs b/src/BenchmarkDotNet/Columns/BaselineCustomColumn.cs index b808d988d8..4dabe8e502 100644 --- a/src/BenchmarkDotNet/Columns/BaselineCustomColumn.cs +++ b/src/BenchmarkDotNet/Columns/BaselineCustomColumn.cs @@ -43,7 +43,7 @@ public abstract string GetValue(Summary summary, BenchmarkCase benchmarkCase, St public override string ToString() => ColumnName; public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false; - internal static bool ResultsAreInvalid(Summary summary, BenchmarkCase benchmarkCase, BenchmarkCase baseline) + internal static bool ResultsAreInvalid(Summary summary, BenchmarkCase benchmarkCase, BenchmarkCase? baseline) { return baseline == null || summary[baseline] == null || diff --git a/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs b/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs index 0677f73c77..60bb629ee8 100644 --- a/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs +++ b/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs @@ -46,7 +46,7 @@ public IEnumerable GetColumns(Summary summary) if (NeedToShow(summary, s => s.Percentiles.P95 > s.Mean + 3 * s.StandardDeviation)) yield return StatisticColumn.P95; if (NeedToShow(summary, s => s.N >= 3 && - (!s.GetConfidenceInterval(ConfidenceLevel.L99, s.N).Contains(s.Median) || + (!s.GetConfidenceInterval(ConfidenceLevel.L99).Contains(s.Median) || Math.Abs(s.Median - s.Mean) > s.Mean * 0.2))) yield return StatisticColumn.Median; if (NeedToShow(summary, s => s.StandardDeviation > 1e-9)) diff --git a/src/BenchmarkDotNet/Columns/MetricColumn.cs b/src/BenchmarkDotNet/Columns/MetricColumn.cs index 645891502a..feb615bfb0 100644 --- a/src/BenchmarkDotNet/Columns/MetricColumn.cs +++ b/src/BenchmarkDotNet/Columns/MetricColumn.cs @@ -1,8 +1,9 @@ using System.Linq; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; -using Perfolizer.Common; using Perfolizer.Horology; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Columns { @@ -26,8 +27,8 @@ public class MetricColumn : IColumn public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false; public bool IsAvailable(Summary summary) => summary.Reports.Any(report => - report.Metrics.TryGetValue(descriptor.Id, out var metric) - && metric.Descriptor.GetIsAvailable(metric)); + report.Metrics.TryGetValue(descriptor.Id, out var metric) + && metric.Descriptor.GetIsAvailable(metric)); public string GetValue(Summary summary, BenchmarkCase benchmarkCase) => GetValue(summary, benchmarkCase, SummaryStyle.Default); @@ -43,18 +44,23 @@ public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyl var cultureInfo = summary.GetCultureInfo(); bool printUnits = style.PrintUnitsInContent || style.PrintUnitsInHeader; - UnitPresentation unitPresentation = UnitPresentation.FromVisibility(style.PrintUnitsInContent); + var unitPresentation = new UnitPresentation(style.PrintUnitsInContent, minUnitWidth: 0, gap: true); + string numberFormat = descriptor.NumberFormat; if (printUnits && descriptor.UnitType == UnitType.CodeSize) - return SizeValue.FromBytes((long) metric.Value).ToString(style.CodeSizeUnit, cultureInfo, descriptor.NumberFormat, unitPresentation); + return SizeValue.FromBytes((long)metric.Value).ToString(style.CodeSizeUnit, numberFormat, cultureInfo, unitPresentation); if (printUnits && descriptor.UnitType == UnitType.Size) - return SizeValue.FromBytes((long) metric.Value).ToString(style.SizeUnit, cultureInfo, descriptor.NumberFormat, unitPresentation); + return SizeValue.FromBytes((long)metric.Value).ToString(style.SizeUnit, numberFormat, cultureInfo, unitPresentation); if (printUnits && descriptor.UnitType == UnitType.Time) - return TimeInterval.FromNanoseconds(metric.Value).ToString(style.TimeUnit, cultureInfo, descriptor.NumberFormat, unitPresentation); + { + if (numberFormat.IsBlank()) + numberFormat = "N4"; + return TimeInterval.FromNanoseconds(metric.Value).ToString(style.TimeUnit, numberFormat, cultureInfo, unitPresentation); + } - return metric.Value.ToString(descriptor.NumberFormat, cultureInfo); + return metric.Value.ToString(numberFormat, cultureInfo); } public override string ToString() => descriptor.DisplayName; } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Columns/SizeUnit.cs b/src/BenchmarkDotNet/Columns/SizeUnit.cs deleted file mode 100644 index b919fb7fa0..0000000000 --- a/src/BenchmarkDotNet/Columns/SizeUnit.cs +++ /dev/null @@ -1,75 +0,0 @@ -using System; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using JetBrains.Annotations; - -namespace BenchmarkDotNet.Columns -{ - [SuppressMessage("ReSharper", "InconsistentNaming")] // We want to use "KB", "MB", "GB", "TB" - public class SizeUnit : IEquatable - { - [PublicAPI] public string Name { get; } - [PublicAPI] public string Description { get; } - [PublicAPI] public long ByteAmount { get; } - - public SizeUnit(string name, string description, long byteAmount) - { - Name = name; - Description = description; - ByteAmount = byteAmount; - } - - private const long BytesInKiloByte = 1024L; // this value MUST NOT be changed - - public SizeValue ToValue(long value = 1) => new SizeValue(value, this); - - [PublicAPI] public static readonly SizeUnit B = new SizeUnit("B", "Byte", 1L); - [PublicAPI] public static readonly SizeUnit KB = new SizeUnit("KB", "Kilobyte", BytesInKiloByte); - [PublicAPI] public static readonly SizeUnit MB = new SizeUnit("MB", "Megabyte", BytesInKiloByte * BytesInKiloByte); - [PublicAPI] public static readonly SizeUnit GB = new SizeUnit("GB", "Gigabyte", BytesInKiloByte * BytesInKiloByte * BytesInKiloByte); - [PublicAPI] public static readonly SizeUnit TB = new SizeUnit("TB", "Terabyte", BytesInKiloByte * BytesInKiloByte * BytesInKiloByte * BytesInKiloByte); - [PublicAPI] public static readonly SizeUnit[] All = { B, KB, MB, GB, TB }; - - public static SizeUnit GetBestSizeUnit(params long[] values) - { - if (!values.Any()) - return B; - // Use the largest unit to display the smallest recorded measurement without loss of precision. - long minValue = values.Min(); - foreach (var sizeUnit in All) - { - if (minValue < sizeUnit.ByteAmount * BytesInKiloByte) - return sizeUnit; - } - return All.Last(); - } - - public static double Convert(long value, SizeUnit from, SizeUnit to) => value * (double)from.ByteAmount / (to ?? GetBestSizeUnit(value)).ByteAmount; - - public bool Equals(SizeUnit other) - { - if (ReferenceEquals(null, other)) - return false; - if (ReferenceEquals(this, other)) - return true; - return string.Equals(Name, other.Name) && string.Equals(Description, other.Description) && ByteAmount == other.ByteAmount; - } - - public override bool Equals(object obj) - { - if (ReferenceEquals(null, obj)) - return false; - if (ReferenceEquals(this, obj)) - return true; - if (obj.GetType() != this.GetType()) - return false; - return Equals((SizeUnit) obj); - } - - public override int GetHashCode() => HashCode.Combine(Name, Description, ByteAmount); - - public static bool operator ==(SizeUnit left, SizeUnit right) => Equals(left, right); - - public static bool operator !=(SizeUnit left, SizeUnit right) => !Equals(left, right); - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Columns/SizeValue.cs b/src/BenchmarkDotNet/Columns/SizeValue.cs deleted file mode 100644 index 9478989eb1..0000000000 --- a/src/BenchmarkDotNet/Columns/SizeValue.cs +++ /dev/null @@ -1,61 +0,0 @@ -using System.Globalization; -using BenchmarkDotNet.Helpers; -using JetBrains.Annotations; -using Perfolizer.Common; - -namespace BenchmarkDotNet.Columns -{ - public struct SizeValue - { - public long Bytes { get; } - - public SizeValue(long bytes) => Bytes = bytes; - - public SizeValue(long bytes, SizeUnit unit) : this(bytes * unit.ByteAmount) { } - - public static readonly SizeValue B = SizeUnit.B.ToValue(); - public static readonly SizeValue KB = SizeUnit.KB.ToValue(); - public static readonly SizeValue MB = SizeUnit.MB.ToValue(); - public static readonly SizeValue GB = SizeUnit.GB.ToValue(); - public static readonly SizeValue TB = SizeUnit.TB.ToValue(); - - [Pure] public static SizeValue FromBytes(long value) => value * B; - [Pure] public static SizeValue FromKilobytes(long value) => value * KB; - [Pure] public static SizeValue FromMegabytes(long value) => value * MB; - [Pure] public static SizeValue FromGigabytes(long value) => value * GB; - [Pure] public static SizeValue FromTerabytes(long value) => value * TB; - - [Pure] public static SizeValue operator *(SizeValue value, long k) => new SizeValue(value.Bytes * k); - [Pure] public static SizeValue operator *(long k, SizeValue value) => new SizeValue(value.Bytes * k); - - [Pure] - public string ToString( - CultureInfo? cultureInfo, - string? format = "0.##", - UnitPresentation? unitPresentation = null) - { - return ToString(null, cultureInfo, format, unitPresentation); - } - - [Pure] - public string ToString( - SizeUnit? sizeUnit, - CultureInfo? cultureInfo, - string? format = "0.##", - UnitPresentation? unitPresentation = null) - { - sizeUnit = sizeUnit ?? SizeUnit.GetBestSizeUnit(Bytes); - cultureInfo = cultureInfo ?? DefaultCultureInfo.Instance; - format = format ?? "0.##"; - unitPresentation = unitPresentation ?? UnitPresentation.Default; - double unitValue = SizeUnit.Convert(Bytes, SizeUnit.B, sizeUnit); - if (unitPresentation.IsVisible) - { - string unitName = sizeUnit.Name.PadLeft(unitPresentation.MinUnitWidth); - return $"{unitValue.ToString(format, cultureInfo)} {unitName}"; - } - - return unitValue.ToString(format, cultureInfo); - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Columns/StatisticColumn.cs b/src/BenchmarkDotNet/Columns/StatisticColumn.cs index f0b9655acc..1e1fd63b35 100644 --- a/src/BenchmarkDotNet/Columns/StatisticColumn.cs +++ b/src/BenchmarkDotNet/Columns/StatisticColumn.cs @@ -6,10 +6,10 @@ using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using JetBrains.Annotations; -using Perfolizer.Common; using Perfolizer.Horology; using Perfolizer.Mathematics.Common; using Perfolizer.Mathematics.Multimodality; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Columns { @@ -38,7 +38,7 @@ private enum Priority s => s.StandardDeviation, Priority.Main, parentColumn: Mean); public static readonly IColumn Error = new StatisticColumn(Column.Error, "Half of 99.9% confidence interval", - s => new ConfidenceInterval(s.Mean, s.StandardError, s.N, ConfidenceLevel.L999).Margin, Priority.Main, parentColumn: Mean); + s => s.GetConfidenceInterval(ConfidenceLevel.L999).Margin, Priority.Main, parentColumn: Mean); public static readonly IColumn OperationsPerSecond = new StatisticColumn(Column.OperationPerSecond, "Operation per second", s => 1.0 * 1000 * 1000 * 1000 / s.Mean, Priority.Additional, UnitType.Dimensionless); @@ -67,7 +67,7 @@ private enum Priority /// See http://www.brendangregg.com/FrequencyTrails/modes.html /// public static readonly IColumn MValue = new StatisticColumn(Column.MValue, "Modal value, see http://www.brendangregg.com/FrequencyTrails/modes.html", - s => MValueCalculator.Calculate(s.OriginalValues), Priority.Additional, UnitType.Dimensionless); + s => MValueCalculator.Calculate(s.Sample.Values), Priority.Additional, UnitType.Dimensionless); public static readonly IColumn Iterations = new StatisticColumn(Column.Iterations, "Number of target iterations", s => s.N, Priority.Additional, UnitType.Dimensionless); @@ -84,17 +84,17 @@ private enum Priority [PublicAPI] public static IColumn CiLower(ConfidenceLevel level) => new StatisticColumn( - $"CI{level.ToPercentStr()} Lower", $"Lower bound of {level.ToPercentStr()} confidence interval", + $"CI{level} Lower", $"Lower bound of {level} confidence interval", s => new ConfidenceInterval(s.Mean, s.StandardError, s.N, level).Lower, Priority.Additional); [PublicAPI] public static IColumn CiUpper(ConfidenceLevel level) => new StatisticColumn( - $"CI{level.ToPercentStr()} Upper", $"Upper bound of {level.ToPercentStr()} confidence interval", + $"CI{level} Upper", $"Upper bound of {level} confidence interval", s => new ConfidenceInterval(s.Mean, s.StandardError, s.N, level).Upper, Priority.Additional); [PublicAPI] public static IColumn CiError(ConfidenceLevel level) => new StatisticColumn( - $"CI{level.ToPercentStr()} Margin", $"Half of {level.ToPercentStr()} confidence interval", + $"CI{level} Margin", $"Half of {level} confidence interval", s => new ConfidenceInterval(s.Mean, s.StandardError, s.N, level).Margin, Priority.Additional); @@ -118,33 +118,34 @@ private StatisticColumn(string columnName, string legend, Func Format(summary, benchmarkCase.Config, summary[benchmarkCase].ResultStatistics, SummaryStyle.Default); + => Format(summary, benchmarkCase.Config, summary[benchmarkCase]?.ResultStatistics, SummaryStyle.Default); public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) - => Format(summary, benchmarkCase.Config, summary[benchmarkCase].ResultStatistics, style); + => Format(summary, benchmarkCase.Config, summary[benchmarkCase]?.ResultStatistics, style); public bool IsAvailable(Summary summary) => true; public bool AlwaysShow => true; public ColumnCategory Category => ColumnCategory.Statistics; - public int PriorityInCategory => (int) priority; + public int PriorityInCategory => (int)priority; public bool IsNumeric => true; public UnitType UnitType { get; } public string Legend { get; } public List GetAllValues(Summary summary, SummaryStyle style) - => summary.Reports + { + return summary.Reports .Where(r => r.ResultStatistics != null) .Select(r => calc(r.ResultStatistics)) .Where(v => !double.IsNaN(v) && !double.IsInfinity(v)) - .Select(v => UnitType == UnitType.Time ? v / style.TimeUnit.NanosecondAmount : v) + .Select(v => UnitType == UnitType.Time && style.TimeUnit != null ? v / style.TimeUnit.BaseUnits : v) .ToList(); + } - private string Format(Summary summary, ImmutableConfig config, Statistics statistics, SummaryStyle style) + private string Format(Summary summary, ImmutableConfig config, Statistics? statistics, SummaryStyle style) { if (statistics == null) return "NA"; - int precision = summary.DisplayPrecisionManager.GetPrecision(style, this, parentColumn); string format = "N" + precision; @@ -155,9 +156,9 @@ private string Format(Summary summary, ImmutableConfig config, Statistics statis ? TimeInterval.FromNanoseconds(value) .ToString( style.TimeUnit, - style.CultureInfo, format, - UnitPresentation.FromVisibility(style.PrintUnitsInContent)) + style.CultureInfo, + new UnitPresentation(style.PrintUnitsInContent, minUnitWidth: 0, gap: true)) : value.ToString(format, style.CultureInfo); } diff --git a/src/BenchmarkDotNet/Columns/StatisticalTestColumn.cs b/src/BenchmarkDotNet/Columns/StatisticalTestColumn.cs index 0316b9c5f4..52a95bc04e 100644 --- a/src/BenchmarkDotNet/Columns/StatisticalTestColumn.cs +++ b/src/BenchmarkDotNet/Columns/StatisticalTestColumn.cs @@ -4,50 +4,57 @@ using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; +using Perfolizer.Mathematics.Common; using Perfolizer.Mathematics.SignificanceTesting; -using Perfolizer.Mathematics.Thresholds; +using Perfolizer.Mathematics.SignificanceTesting.MannWhitney; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Columns { - public class StatisticalTestColumn : BaselineCustomColumn + public class StatisticalTestColumn(Threshold threshold, SignificanceLevel? significanceLevel = null) : BaselineCustomColumn { - public static StatisticalTestColumn Create(StatisticalTestKind kind, Threshold threshold, bool showPValues = false) - => new StatisticalTestColumn(kind, threshold, showPValues); + private static readonly SignificanceLevel DefaultSignificanceLevel = SignificanceLevel.P1E5; - public StatisticalTestKind Kind { get; } - public Threshold Threshold { get; } - public bool ShowPValues { get; } + public static StatisticalTestColumn CreateDefault() => new(new PercentValue(10).ToThreshold()); - public StatisticalTestColumn(StatisticalTestKind kind, Threshold threshold, bool showPValues = false) + public static StatisticalTestColumn Create(Threshold threshold, SignificanceLevel? significanceLevel = null) => new(threshold, significanceLevel); + + public static StatisticalTestColumn Create(string threshold, SignificanceLevel? significanceLevel = null) { - Kind = kind; - Threshold = threshold; - ShowPValues = showPValues; + if (!Threshold.TryParse(threshold, out var parsedThreshold)) + throw new ArgumentException($"Can't parse threshold '{threshold}'"); + return new StatisticalTestColumn(parsedThreshold, significanceLevel); } - public override string Id => nameof(StatisticalTestColumn) + "." + Kind + "." + Threshold + "." + (ShowPValues ? "WithDetails" : "WithoutDetails"); - public override string ColumnName => $"{Kind}({Threshold.ToString().Replace(" ", "")}){(ShowPValues ? "/p-values" : "")}"; + public Threshold Threshold { get; } = threshold; + public SignificanceLevel SignificanceLevel { get; } = significanceLevel ?? DefaultSignificanceLevel; + + public override string Id => $"{nameof(StatisticalTestColumn)}/{Threshold}"; + public override string ColumnName => $"MannWhitney({Threshold})"; public override string GetValue(Summary summary, BenchmarkCase benchmarkCase, Statistics baseline, IReadOnlyDictionary baselineMetrics, Statistics current, IReadOnlyDictionary currentMetrics, bool isBaseline) { - var x = baseline.OriginalValues.ToArray(); - var y = current.OriginalValues.ToArray(); - switch (Kind) + if (baseline.Sample.Values.SequenceEqual(current.Sample.Values)) + return "Baseline"; + if (current.Sample.Size == 1 && baseline.Sample.Size == 1) + return "?"; + + var test = new SimpleEquivalenceTest(MannWhitneyTest.Instance); + var comparisonResult = test.Perform(current.Sample, baseline.Sample, Threshold, SignificanceLevel); + return comparisonResult switch { - case StatisticalTestKind.Welch: - return StatisticalTestHelper.CalculateTost(WelchTest.Instance, x, y, Threshold).ToString(ShowPValues); - case StatisticalTestKind.MannWhitney: - return StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, x, y, Threshold).ToString(ShowPValues); - default: - throw new ArgumentOutOfRangeException(); - } + ComparisonResult.Greater => "Slower", + ComparisonResult.Indistinguishable => "Same", + ComparisonResult.Lesser => "Faster", + _ => throw new ArgumentOutOfRangeException() + }; } - public override int PriorityInCategory => (int) Kind; + public override int PriorityInCategory => 0; public override bool IsNumeric => false; public override UnitType UnitType => UnitType.Dimensionless; - public override string Legend => $"{Kind}-based TOST equivalence test with {Threshold} threshold{(ShowPValues ? ". Format: 'Result: p-value(Slower)|p-value(Faster)'" : "")}"; + public override string Legend => $"MannWhitney-based equivalence test (threshold={Threshold}, alpha = {SignificanceLevel})"; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Columns/TagColumn.cs b/src/BenchmarkDotNet/Columns/TagColumn.cs index e4300f037c..425b4d2ad8 100644 --- a/src/BenchmarkDotNet/Columns/TagColumn.cs +++ b/src/BenchmarkDotNet/Columns/TagColumn.cs @@ -19,7 +19,7 @@ public TagColumn(string columnName, Func getTag) } public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false; - public string GetValue(Summary summary, BenchmarkCase benchmarkCase) => getTag(benchmarkCase.Descriptor.WorkloadMethod.Name); + public string GetValue(Summary summary, BenchmarkCase benchmarkCase) => getTag(benchmarkCase.Descriptor?.WorkloadMethod?.Name ?? ""); public bool IsAvailable(Summary summary) => true; public bool AlwaysShow => true; diff --git a/src/BenchmarkDotNet/Columns/UnitType.cs b/src/BenchmarkDotNet/Columns/UnitType.cs index 606a68bd2d..f9e7815fc4 100644 --- a/src/BenchmarkDotNet/Columns/UnitType.cs +++ b/src/BenchmarkDotNet/Columns/UnitType.cs @@ -1,5 +1,6 @@ namespace BenchmarkDotNet.Columns { + // TODO: migrate to Perfolizer.Metrology public enum UnitType { Dimensionless, diff --git a/src/BenchmarkDotNet/Configs/ConfigOptions.cs b/src/BenchmarkDotNet/Configs/ConfigOptions.cs index c1ad75d642..06bc7768c7 100644 --- a/src/BenchmarkDotNet/Configs/ConfigOptions.cs +++ b/src/BenchmarkDotNet/Configs/ConfigOptions.cs @@ -48,7 +48,11 @@ public enum ConfigOptions /// /// Continue the execution if the last run was stopped. /// - Resume = 1 << 9 + Resume = 1 << 9, + /// + /// Determines whether parallel build of benchmark projects should be disabled. + /// + DisableParallelBuild = 1 << 10, } internal static class ConfigOptionsExtensions diff --git a/src/BenchmarkDotNet/Configs/DebugConfig.cs b/src/BenchmarkDotNet/Configs/DebugConfig.cs index 0fdda7b08d..551c38fa09 100644 --- a/src/BenchmarkDotNet/Configs/DebugConfig.cs +++ b/src/BenchmarkDotNet/Configs/DebugConfig.cs @@ -71,6 +71,7 @@ public abstract class DebugConfig : IConfig public SummaryStyle SummaryStyle => SummaryStyle.Default; public ConfigUnionRule UnionRule => ConfigUnionRule.Union; public TimeSpan BuildTimeout => DefaultConfig.Instance.BuildTimeout; + public WakeLockType WakeLock => WakeLockType.None; public string ArtifactsPath => null; // DefaultConfig.ArtifactsPath will be used if the user does not specify it in explicit way diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs index 2323d4fdc4..d188b67415 100644 --- a/src/BenchmarkDotNet/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet/Configs/DefaultConfig.cs @@ -4,6 +4,7 @@ using System.IO; using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.EventProcessors; using BenchmarkDotNet.Exporters; @@ -71,6 +72,7 @@ public IEnumerable GetValidators() yield return DeferredExecutionValidator.FailOnError; yield return ParamsAllValuesValidator.FailOnError; yield return ParamsValidator.FailOnError; + yield return RuntimeValidator.DontFailOnError; } public IOrderer Orderer => null; @@ -86,11 +88,13 @@ public IEnumerable GetValidators() public TimeSpan BuildTimeout => TimeSpan.FromSeconds(120); + public WakeLockType WakeLock => WakeLockType.System; + public string ArtifactsPath { get { - var root = RuntimeInformation.IsAndroid() ? + var root = OsDetector.IsAndroid() ? Environment.GetFolderPath(Environment.SpecialFolder.UserProfile) : Directory.GetCurrentDirectory(); return Path.Combine(root, "BenchmarkDotNet.Artifacts"); diff --git a/src/BenchmarkDotNet/Configs/IConfig.cs b/src/BenchmarkDotNet/Configs/IConfig.cs index b311c235f5..9e3bf50ce6 100644 --- a/src/BenchmarkDotNet/Configs/IConfig.cs +++ b/src/BenchmarkDotNet/Configs/IConfig.cs @@ -55,6 +55,8 @@ public interface IConfig /// TimeSpan BuildTimeout { get; } + public WakeLockType WakeLock { get; } + /// /// Collect any errors or warnings when composing the configuration /// diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs index 1216ef3b16..0043ecbe4b 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs @@ -56,6 +56,7 @@ internal ImmutableConfig( SummaryStyle summaryStyle, ConfigOptions options, TimeSpan buildTimeout, + WakeLockType wakeLock, IReadOnlyList configAnalysisConclusion) { columnProviders = uniqueColumnProviders; @@ -78,6 +79,7 @@ internal ImmutableConfig( SummaryStyle = summaryStyle; Options = options; BuildTimeout = buildTimeout; + WakeLock = wakeLock; ConfigAnalysisConclusion = configAnalysisConclusion; } @@ -89,6 +91,7 @@ internal ImmutableConfig( public ICategoryDiscoverer CategoryDiscoverer { get; } public SummaryStyle SummaryStyle { get; } public TimeSpan BuildTimeout { get; } + public WakeLockType WakeLock { get; } public IEnumerable GetColumnProviders() => columnProviders; public IEnumerable GetExporters() => exporters; @@ -119,7 +122,7 @@ internal ImmutableConfig( public bool HasExtraStatsDiagnoser() => HasMemoryDiagnoser() || HasThreadingDiagnoser() || HasExceptionDiagnoser(); - public IDiagnoser GetCompositeDiagnoser(BenchmarkCase benchmarkCase, RunMode runMode) + public IDiagnoser? GetCompositeDiagnoser(BenchmarkCase benchmarkCase, RunMode runMode) { var diagnosersForGivenMode = diagnosers.Where(diagnoser => diagnoser.GetRunMode(benchmarkCase) == runMode).ToImmutableHashSet(); diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index d7c0b5eb0f..f93e5590d0 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -76,6 +76,7 @@ public static ImmutableConfig Create(IConfig source) source.SummaryStyle ?? SummaryStyle.Default, source.Options, source.BuildTimeout, + source.WakeLock, configAnalyse.AsReadOnly() ); } diff --git a/src/BenchmarkDotNet/Configs/ManualConfig.cs b/src/BenchmarkDotNet/Configs/ManualConfig.cs index 5ea1be24e9..4ececd28f1 100644 --- a/src/BenchmarkDotNet/Configs/ManualConfig.cs +++ b/src/BenchmarkDotNet/Configs/ManualConfig.cs @@ -58,6 +58,7 @@ public class ManualConfig : IConfig [PublicAPI] public ICategoryDiscoverer CategoryDiscoverer { get; set; } [PublicAPI] public SummaryStyle SummaryStyle { get; set; } [PublicAPI] public TimeSpan BuildTimeout { get; set; } = DefaultConfig.Instance.BuildTimeout; + [PublicAPI] public WakeLockType WakeLock { get; set; } = DefaultConfig.Instance.WakeLock; public IReadOnlyList ConfigAnalysisConclusion => emptyConclusion; @@ -109,6 +110,12 @@ public ManualConfig WithBuildTimeout(TimeSpan buildTimeout) return this; } + public ManualConfig WithWakeLock(WakeLockType wakeLockType) + { + WakeLock = wakeLockType; + return this; + } + [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete("This method will soon be removed, please start using .AddColumn() instead.")] public void Add(params IColumn[] newColumns) => AddColumn(newColumns); @@ -273,6 +280,7 @@ public void Add(IConfig config) columnHidingRules.AddRange(config.GetColumnHidingRules()); Options |= config.Options; BuildTimeout = GetBuildTimeout(BuildTimeout, config.BuildTimeout); + WakeLock = GetWakeLock(WakeLock, config.WakeLock); } /// @@ -319,6 +327,12 @@ public static ManualConfig Union(IConfig globalConfig, IConfig localConfig) return manualConfig; } + internal ManualConfig RemoveLoggersOfType() + { + loggers.RemoveAll(logger => logger is T); + return this; + } + internal void RemoveAllJobs() => jobs.Clear(); internal void RemoveAllDiagnosers() => diagnosers.Clear(); @@ -327,5 +341,12 @@ private static TimeSpan GetBuildTimeout(TimeSpan current, TimeSpan other) => current == DefaultConfig.Instance.BuildTimeout ? other : TimeSpan.FromMilliseconds(Math.Max(current.TotalMilliseconds, other.TotalMilliseconds)); + + private static WakeLockType GetWakeLock(WakeLockType current, WakeLockType other) + { + if (current == DefaultConfig.Instance.WakeLock) { return other; } + if (other == DefaultConfig.Instance.WakeLock) { return current; } + return current.CompareTo(other) > 0 ? current : other; + } } } diff --git a/src/BenchmarkDotNet/Configs/WakeLockType.cs b/src/BenchmarkDotNet/Configs/WakeLockType.cs new file mode 100644 index 0000000000..f547e44764 --- /dev/null +++ b/src/BenchmarkDotNet/Configs/WakeLockType.cs @@ -0,0 +1,20 @@ +namespace BenchmarkDotNet.Configs +{ + public enum WakeLockType + { + /// + /// Allows the system to enter sleep and/or turn off the display while benchmarks are running. + /// + None, + + /// + /// Forces the system to be in the working state while benchmarks are running. + /// + System, + + /// + /// Forces the display to be on while benchmarks are running. + /// + Display + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index e3aed1fedd..6c7b48b0e2 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -2,6 +2,7 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; +using BenchmarkDotNet.Configs; using BenchmarkDotNet.ConsoleArguments.ListBenchmarks; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; @@ -179,6 +180,9 @@ public bool UseDisassemblyDiagnoser [Option("buildTimeout", Required = false, HelpText = "Build timeout in seconds.")] public int? TimeOutInSeconds { get; set; } + [Option("wakeLock", Required = false, HelpText = "Prevents the system from entering sleep or turning off the display. None/System/Display.")] + public WakeLockType? WakeLock { get; set; } + [Option("stopOnFirstError", Required = false, Default = false, HelpText = "Stop on first error.")] public bool StopOnFirstError { get; set; } @@ -237,8 +241,8 @@ public static IEnumerable Examples yield return new Example("Use Job.ShortRun for running the benchmarks", shortName, new CommandLineOptions { BaseJob = "short" }); yield return new Example("Run benchmarks in process", shortName, new CommandLineOptions { RunInProcess = true }); - yield return new Example("Run benchmarks for .NET 4.7.2, .NET Core 2.1 and Mono. .NET 4.7.2 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = new[] { "net472", "netcoreapp2.1", "Mono" } }); - yield return new Example("Run benchmarks for .NET Core 2.0, .NET Core 2.1 and .NET Core 2.2. .NET Core 2.0 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = new[] { "netcoreapp2.0", "netcoreapp2.1", "netcoreapp2.2" } }); + yield return new Example("Run benchmarks for .NET 4.7.2, .NET 8.0 and Mono. .NET 4.7.2 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = new[] { "net472", "net8.0", "Mono" } }); + yield return new Example("Run benchmarks for .NET Core 3.1, .NET 6.0 and .NET 8.0. .NET Core 3.1 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = new[] { "netcoreapp3.1", "net6.0", "net8.0" } }); yield return new Example("Use MemoryDiagnoser to get GC stats", shortName, new CommandLineOptions { UseMemoryDiagnoser = true }); yield return new Example("Use DisassemblyDiagnoser to get disassembly", shortName, new CommandLineOptions { UseDisassemblyDiagnoser = true }); yield return new Example("Use HardwareCountersDiagnoser to get hardware counter info", longName, new CommandLineOptions { HardwareCounters = new[] { nameof(HardwareCounter.CacheMisses), nameof(HardwareCounter.InstructionRetired) } }); @@ -250,8 +254,8 @@ public static IEnumerable Examples yield return new Example("Run selected benchmarks once per iteration", longName, new CommandLineOptions { RunOncePerIteration = true }); yield return new Example("Run selected benchmarks 100 times per iteration. Perform single warmup iteration and 5 actual workload iterations", longName, new CommandLineOptions { InvocationCount = 100, WarmupIterationCount = 1, IterationCount = 5}); yield return new Example("Run selected benchmarks 250ms per iteration. Perform from 9 to 15 iterations", longName, new CommandLineOptions { IterationTimeInMilliseconds = 250, MinIterationCount = 9, MaxIterationCount = 15}); - yield return new Example("Run MannWhitney test with relative ratio of 5% for all benchmarks for .NET Core 2.0 (base) vs .NET Core 2.1 (diff). .NET Core 2.0 will be baseline because it was provided as first.", longName, - new CommandLineOptions { Filters = new[] { "*"}, Runtimes = new[] { "netcoreapp2.0", "netcoreapp2.1" }, StatisticalTestThreshold = "5%" }); + yield return new Example("Run MannWhitney test with relative ratio of 5% for all benchmarks for .NET 6.0 (base) vs .NET 8.0 (diff). .NET Core 6.0 will be baseline because it was provided as first.", longName, + new CommandLineOptions { Filters = new[] { "*"}, Runtimes = new[] { "net6.0", "net8.0" }, StatisticalTestThreshold = "5%" }); yield return new Example("Run benchmarks using environment variables 'ENV_VAR_KEY_1' with value 'value_1' and 'ENV_VAR_KEY_2' with value 'value_2'", longName, new CommandLineOptions { EnvironmentVariables = new[] { "ENV_VAR_KEY_1:value_1", "ENV_VAR_KEY_2:value_2" } }); yield return new Example("Hide Mean and Ratio columns (use double quotes for multi-word columns: \"Alloc Ratio\")", shortName, new CommandLineOptions { HiddenColumns = new[] { "Mean", "Ratio" }, }); diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 0a125900f4..ac61ddb5f1 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -1,775 +1,797 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.IO; -using System.Linq; -using System.Text; -using BenchmarkDotNet.Columns; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Exporters; -using BenchmarkDotNet.Exporters.Csv; -using BenchmarkDotNet.Exporters.Json; -using BenchmarkDotNet.Exporters.Xml; -using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Filters; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Toolchains.CoreRun; -using BenchmarkDotNet.Toolchains.CsProj; -using BenchmarkDotNet.Toolchains.DotNetCli; -using BenchmarkDotNet.Toolchains.InProcess.Emit; -using BenchmarkDotNet.Toolchains.MonoAotLLVM; -using BenchmarkDotNet.Toolchains.MonoWasm; -using BenchmarkDotNet.Toolchains.NativeAot; -using CommandLine; -using Perfolizer.Horology; -using Perfolizer.Mathematics.OutlierDetection; -using Perfolizer.Mathematics.SignificanceTesting; -using Perfolizer.Mathematics.Thresholds; -using BenchmarkDotNet.Toolchains.Mono; - -namespace BenchmarkDotNet.ConsoleArguments -{ - public static class ConfigParser - { - private const int MinimumDisplayWidth = 80; - private const char EnvVarKeyValueSeparator = ':'; - - private static readonly IReadOnlyDictionary AvailableJobs = new Dictionary(StringComparer.InvariantCultureIgnoreCase) - { - { "default", Job.Default }, - { "dry", Job.Dry }, - { "short", Job.ShortRun }, - { "medium", Job.MediumRun }, - { "long", Job.LongRun }, - { "verylong", Job.VeryLongRun } - }; - - [SuppressMessage("ReSharper", "StringLiteralTypo")] - [SuppressMessage("ReSharper", "CoVariantArrayConversion")] - private static readonly IReadOnlyDictionary AvailableExporters = - new Dictionary(StringComparer.InvariantCultureIgnoreCase) - { - { "csv", new[] { CsvExporter.Default } }, - { "csvmeasurements", new[] { CsvMeasurementsExporter.Default } }, - { "html", new[] { HtmlExporter.Default } }, - { "markdown", new[] { MarkdownExporter.Default } }, - { "atlassian", new[] { MarkdownExporter.Atlassian } }, - { "stackoverflow", new[] { MarkdownExporter.StackOverflow } }, - { "github", new[] { MarkdownExporter.GitHub } }, - { "plain", new[] { PlainExporter.Default } }, - { "rplot", new[] { CsvMeasurementsExporter.Default, RPlotExporter.Default } }, // R Plots depends on having the full measurements available - { "json", new[] { JsonExporter.Default } }, - { "briefjson", new[] { JsonExporter.Brief } }, - { "fulljson", new[] { JsonExporter.Full } }, - { "asciidoc", new[] { AsciiDocExporter.Default } }, - { "xml", new[] { XmlExporter.Default } }, - { "briefxml", new[] { XmlExporter.Brief } }, - { "fullxml", new[] { XmlExporter.Full } } - }; - - public static (bool isSuccess, IConfig config, CommandLineOptions options) Parse(string[] args, ILogger logger, IConfig? globalConfig = null) - { - (bool isSuccess, IConfig config, CommandLineOptions options) result = default; - - var (expandSuccess, expandedArgs) = ExpandResponseFile(args, logger); - if (!expandSuccess) - { - return (false, default, default); - } - - args = expandedArgs; - using (var parser = CreateParser(logger)) - { - parser - .ParseArguments(args) - .WithParsed(options => result = Validate(options, logger) ? (true, CreateConfig(options, globalConfig, args), options) : (false, default, default)) - .WithNotParsed(errors => result = (false, default, default)); - } - - return result; - } - - private static (bool Success, string[] ExpandedTokens) ExpandResponseFile(string[] args, ILogger logger) - { - List result = new (); - foreach (var arg in args) - { - if (arg.StartsWith("@")) - { - var fileName = arg.Substring(1); - try - { - if (File.Exists(fileName)) - { - var lines = File.ReadAllLines(fileName); - foreach (var line in lines) - { - result.AddRange(ConsumeTokens(line)); - } - } - else - { - logger.WriteLineError($"Response file {fileName} does not exists."); - return (false, Array.Empty()); - } - } - catch (Exception ex) - { - logger.WriteLineError($"Failed to parse RSP file: {fileName}, {ex.Message}"); - return (false, Array.Empty()); - } - } - else - { - result.Add(arg); - } - } - - return (true, result.ToArray()); - } - - private static IEnumerable ConsumeTokens(string line) - { - bool insideQuotes = false; - var token = new StringBuilder(); - for (int i = 0; i < line.Length; i++) - { - char currentChar = line[i]; - if (currentChar == ' ' && !insideQuotes) - { - if (token.Length > 0) - { - yield return GetToken(); - token = new StringBuilder(); - } - - continue; - } - - if (currentChar == '"') - { - insideQuotes = !insideQuotes; - continue; - } - - if (currentChar == '\\' && insideQuotes) - { - if (line[i + 1] == '"') - { - insideQuotes = false; - i++; - continue; - } - - if (line[i + 1] == '\\') - { - token.Append('\\'); - i++; - continue; - } - } - - token.Append(currentChar); - } - - if (token.Length > 0) - { - yield return GetToken(); - } - - string GetToken() - { - var result = token.ToString(); - if (result.Contains(' ')) - { - // Workaround for CommandLine library issue with parsing these kind of args. - return " " + result; - } - - return result; - } - } - - internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Action updater) - { - (bool isSuccess, CommandLineOptions options) result = default; - - ILogger logger = NullLogger.Instance; - using (var parser = CreateParser(logger)) - { - parser - .ParseArguments(args) - .WithParsed(options => result = Validate(options, logger) ? (true, options) : (false, default)) - .WithNotParsed(errors => result = (false, default)); - - if (!result.isSuccess) - { - updatedArgs = null; - return false; - } - - updater(result.options); - - updatedArgs = parser.FormatCommandLine(result.options, settings => settings.SkipDefault = true).Split(); - return true; - } - } - - private static Parser CreateParser(ILogger logger) - => new Parser(settings => - { - settings.CaseInsensitiveEnumValues = true; - settings.CaseSensitive = false; - settings.EnableDashDash = true; - settings.IgnoreUnknownArguments = false; - settings.HelpWriter = new LoggerWrapper(logger); - settings.MaximumDisplayWidth = Math.Max(MinimumDisplayWidth, GetMaximumDisplayWidth()); - }); - - private static bool Validate(CommandLineOptions options, ILogger logger) - { - if (!AvailableJobs.ContainsKey(options.BaseJob)) - { - logger.WriteLineError($"The provided base job \"{options.BaseJob}\" is invalid. Available options are: {string.Join(", ", AvailableJobs.Keys)}."); - return false; - } - - foreach (string runtime in options.Runtimes) - { - if (!TryParse(runtime, out RuntimeMoniker runtimeMoniker)) - { - logger.WriteLineError($"The provided runtime \"{runtime}\" is invalid. Available options are: {string.Join(", ", Enum.GetNames(typeof(RuntimeMoniker)).Select(name => name.ToLower()))}."); - return false; - } - else if (runtimeMoniker == RuntimeMoniker.MonoAOTLLVM && (options.AOTCompilerPath == null || options.AOTCompilerPath.IsNotNullButDoesNotExist())) - { - logger.WriteLineError($"The provided {nameof(options.AOTCompilerPath)} \"{options.AOTCompilerPath}\" does NOT exist. It MUST be provided."); - } - } - - foreach (string exporter in options.Exporters) - if (!AvailableExporters.ContainsKey(exporter)) - { - logger.WriteLineError($"The provided exporter \"{exporter}\" is invalid. Available options are: {string.Join(", ", AvailableExporters.Keys)}."); - return false; - } - - if (options.CliPath.IsNotNullButDoesNotExist()) - { - logger.WriteLineError($"The provided {nameof(options.CliPath)} \"{options.CliPath}\" does NOT exist."); - return false; - } - - foreach (var coreRunPath in options.CoreRunPaths) - if (coreRunPath.IsNotNullButDoesNotExist()) - { - if (Directory.Exists(coreRunPath.FullName)) - { - logger.WriteLineError($"The provided path to CoreRun: \"{coreRunPath}\" exists but it's a directory, not an executable. You need to include CoreRun.exe (corerun on Unix) in the path."); - } - else - { - logger.WriteLineError($"The provided path to CoreRun: \"{coreRunPath}\" does NOT exist."); - } - - return false; - } - - if (options.MonoPath.IsNotNullButDoesNotExist()) - { - logger.WriteLineError($"The provided {nameof(options.MonoPath)} \"{options.MonoPath}\" does NOT exist."); - return false; - } - - if (options.WasmJavascriptEngine.IsNotNullButDoesNotExist()) - { - logger.WriteLineError($"The provided {nameof(options.WasmJavascriptEngine)} \"{options.WasmJavascriptEngine}\" does NOT exist."); - return false; - } - - if (options.IlcPackages.IsNotNullButDoesNotExist()) - { - logger.WriteLineError($"The provided {nameof(options.IlcPackages)} \"{options.IlcPackages}\" does NOT exist."); - return false; - } - - if (options.HardwareCounters.Count() > 3) - { - logger.WriteLineError("You can't use more than 3 HardwareCounters at the same time."); - return false; - } - - foreach (var counterName in options.HardwareCounters) - if (!Enum.TryParse(counterName, ignoreCase: true, out HardwareCounter _)) - { - logger.WriteLineError($"The provided hardware counter \"{counterName}\" is invalid. Available options are: {string.Join("+", Enum.GetNames(typeof(HardwareCounter)))}."); - return false; - } - - if (!string.IsNullOrEmpty(options.StatisticalTestThreshold) && !Threshold.TryParse(options.StatisticalTestThreshold, out _)) - { - logger.WriteLineError("Invalid Threshold for Statistical Test. Use --help to see examples."); - return false; - } - - if (options.EnvironmentVariables.Any(envVar => envVar.IndexOf(EnvVarKeyValueSeparator) <= 0)) - { - logger.WriteLineError($"Environment variable value must be separated from the key using '{EnvVarKeyValueSeparator}'. Use --help to see examples."); - return false; - } - - return true; - } - - private static IConfig CreateConfig(CommandLineOptions options, IConfig globalConfig, string[] args) - { - var config = new ManualConfig(); - - var baseJob = GetBaseJob(options, globalConfig); - var expanded = Expand(baseJob.UnfreezeCopy(), options, args).ToArray(); // UnfreezeCopy ensures that each of the expanded jobs will have it's own ID - if (expanded.Length > 1) - expanded[0] = expanded[0].AsBaseline(); // if the user provides multiple jobs, then the first one should be a baseline - config.AddJob(expanded); - if (config.GetJobs().IsEmpty() && baseJob != Job.Default) - config.AddJob(baseJob); - - config.AddExporter(options.Exporters.SelectMany(exporter => AvailableExporters[exporter]).ToArray()); - - config.AddHardwareCounters(options.HardwareCounters - .Select(counterName => (HardwareCounter)Enum.Parse(typeof(HardwareCounter), counterName, ignoreCase: true)) - .ToArray()); - - if (options.UseMemoryDiagnoser) - config.AddDiagnoser(MemoryDiagnoser.Default); - if (options.UseThreadingDiagnoser) - config.AddDiagnoser(ThreadingDiagnoser.Default); - if (options.UseExceptionDiagnoser) - config.AddDiagnoser(ExceptionDiagnoser.Default); - if (options.UseDisassemblyDiagnoser) - config.AddDiagnoser(new DisassemblyDiagnoser(new DisassemblyDiagnoserConfig( - maxDepth: options.DisassemblerRecursiveDepth, - filters: options.DisassemblerFilters.ToArray(), - exportDiff: options.DisassemblerDiff))); - if (!string.IsNullOrEmpty(options.Profiler)) - config.AddDiagnoser(DiagnosersLoader.GetImplementation(profiler => profiler.ShortName.EqualsWithIgnoreCase(options.Profiler))); - - if (options.DisplayAllStatistics) - config.AddColumn(StatisticColumn.AllStatistics); - if (!string.IsNullOrEmpty(options.StatisticalTestThreshold) && Threshold.TryParse(options.StatisticalTestThreshold, out var threshold)) - config.AddColumn(new StatisticalTestColumn(StatisticalTestKind.MannWhitney, threshold)); - - if (options.ArtifactsDirectory != null) - config.ArtifactsPath = options.ArtifactsDirectory.FullName; - - var filters = GetFilters(options).ToArray(); - if (filters.Length > 1) - config.AddFilter(new UnionFilter(filters)); - else - config.AddFilter(filters); - - config.HideColumns(options.HiddenColumns.ToArray()); - - config.WithOption(ConfigOptions.JoinSummary, options.Join); - config.WithOption(ConfigOptions.KeepBenchmarkFiles, options.KeepBenchmarkFiles); - config.WithOption(ConfigOptions.DontOverwriteResults, options.DontOverwriteResults); - config.WithOption(ConfigOptions.StopOnFirstError, options.StopOnFirstError); - config.WithOption(ConfigOptions.DisableLogFile, options.DisableLogFile); - config.WithOption(ConfigOptions.LogBuildOutput, options.LogBuildOutput); - config.WithOption(ConfigOptions.GenerateMSBuildBinLog, options.GenerateMSBuildBinLog); - config.WithOption(ConfigOptions.ApplesToApples, options.ApplesToApples); - config.WithOption(ConfigOptions.Resume, options.Resume); - - if (config.Options.IsSet(ConfigOptions.GenerateMSBuildBinLog)) - config.Options |= ConfigOptions.KeepBenchmarkFiles; - - if (options.MaxParameterColumnWidth.HasValue) - config.WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(options.MaxParameterColumnWidth.Value)); - - if (options.TimeOutInSeconds.HasValue) - config.WithBuildTimeout(TimeSpan.FromSeconds(options.TimeOutInSeconds.Value)); - - return config; - } - - private static Job GetBaseJob(CommandLineOptions options, IConfig globalConfig) - { - var baseJob = - globalConfig?.GetJobs().SingleOrDefault(job => job.Meta.IsDefault) // global config might define single custom Default job - ?? AvailableJobs[options.BaseJob.ToLowerInvariant()]; - - if (baseJob != Job.Dry && options.Outliers != OutlierMode.RemoveUpper) - baseJob = baseJob.WithOutlierMode(options.Outliers); - - if (options.Affinity.HasValue) - baseJob = baseJob.WithAffinity((IntPtr)options.Affinity.Value); - - if (options.LaunchCount.HasValue) - baseJob = baseJob.WithLaunchCount(options.LaunchCount.Value); - if (options.WarmupIterationCount.HasValue) - baseJob = baseJob.WithWarmupCount(options.WarmupIterationCount.Value); - if (options.MinWarmupIterationCount.HasValue) - baseJob = baseJob.WithMinWarmupCount(options.MinWarmupIterationCount.Value); - if (options.MaxWarmupIterationCount.HasValue) - baseJob = baseJob.WithMaxWarmupCount(options.MaxWarmupIterationCount.Value); - if (options.IterationTimeInMilliseconds.HasValue) - baseJob = baseJob.WithIterationTime(TimeInterval.FromMilliseconds(options.IterationTimeInMilliseconds.Value)); - if (options.IterationCount.HasValue) - baseJob = baseJob.WithIterationCount(options.IterationCount.Value); - if (options.MinIterationCount.HasValue) - baseJob = baseJob.WithMinIterationCount(options.MinIterationCount.Value); - if (options.MaxIterationCount.HasValue) - baseJob = baseJob.WithMaxIterationCount(options.MaxIterationCount.Value); - if (options.InvocationCount.HasValue) - baseJob = baseJob.WithInvocationCount(options.InvocationCount.Value); - if (options.UnrollFactor.HasValue) - baseJob = baseJob.WithUnrollFactor(options.UnrollFactor.Value); - if (options.RunStrategy.HasValue) - baseJob = baseJob.WithStrategy(options.RunStrategy.Value); - if (options.Platform.HasValue) - baseJob = baseJob.WithPlatform(options.Platform.Value); - if (options.RunOncePerIteration) - baseJob = baseJob.RunOncePerIteration(); - if (options.MemoryRandomization) - baseJob = baseJob.WithMemoryRandomization(); - if (options.NoForcedGCs) - baseJob = baseJob.WithGcForce(false); - if (options.NoEvaluationOverhead) - baseJob = baseJob.WithEvaluateOverhead(false); - - if (options.EnvironmentVariables.Any()) - { - baseJob = baseJob.WithEnvironmentVariables(options.EnvironmentVariables.Select(text => - { - var separated = text.Split(new[] { EnvVarKeyValueSeparator }, 2); - return new EnvironmentVariable(separated[0], separated[1]); - }).ToArray()); - } - - if (AvailableJobs.Values.Contains(baseJob)) // no custom settings - return baseJob; - - return baseJob - .AsDefault(false) // after applying all settings from console args the base job is not default anymore - .AsMutator(); // we mark it as mutator so it will be applied to other jobs defined via attributes and merged later in GetRunnableJobs method - } - - private static IEnumerable Expand(Job baseJob, CommandLineOptions options, string[] args) - { - if (options.RunInProcess) - { - yield return baseJob.WithToolchain(InProcessEmitToolchain.Instance); - } - else if (!string.IsNullOrEmpty(options.ClrVersion)) - { - yield return baseJob.WithRuntime(ClrRuntime.CreateForLocalFullNetFrameworkBuild(options.ClrVersion)); // local builds of .NET Runtime - } - else if (options.CliPath != null && options.Runtimes.IsEmpty() && options.CoreRunPaths.IsEmpty()) - { - yield return CreateCoreJobWithCli(baseJob, options); - } - else - { - // in case both --runtimes and --corerun are specified, the first one is returned first and becomes a baseline job - string first = args.FirstOrDefault(arg => - arg.Equals("--runtimes", StringComparison.OrdinalIgnoreCase) - || arg.Equals("-r", StringComparison.OrdinalIgnoreCase) - - || arg.Equals("--corerun", StringComparison.OrdinalIgnoreCase)); - - if (first is null || first.Equals("--corerun", StringComparison.OrdinalIgnoreCase)) - { - foreach (var coreRunPath in options.CoreRunPaths) - yield return CreateCoreRunJob(baseJob, options, coreRunPath); // local dotnet/runtime builds - - foreach (string runtime in options.Runtimes) // known runtimes - yield return CreateJobForGivenRuntime(baseJob, runtime, options); - } - else - { - foreach (string runtime in options.Runtimes) // known runtimes - yield return CreateJobForGivenRuntime(baseJob, runtime, options); - - foreach (var coreRunPath in options.CoreRunPaths) - yield return CreateCoreRunJob(baseJob, options, coreRunPath); // local dotnet/runtime builds - } - } - } - - private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, CommandLineOptions options) - { - if (!TryParse(runtimeId, out RuntimeMoniker runtimeMoniker)) - { - throw new InvalidOperationException("Impossible, already validated by the Validate method"); - } - - switch (runtimeMoniker) - { - case RuntimeMoniker.Net461: - case RuntimeMoniker.Net462: - case RuntimeMoniker.Net47: - case RuntimeMoniker.Net471: - case RuntimeMoniker.Net472: - case RuntimeMoniker.Net48: - case RuntimeMoniker.Net481: - return baseJob - .WithRuntime(runtimeMoniker.GetRuntime()) - .WithToolchain(CsProjClassicNetToolchain.From(runtimeId, options.RestorePath?.FullName, options.CliPath?.FullName)); - - case RuntimeMoniker.NetCoreApp20: - case RuntimeMoniker.NetCoreApp21: - case RuntimeMoniker.NetCoreApp22: - case RuntimeMoniker.NetCoreApp30: - case RuntimeMoniker.NetCoreApp31: -#pragma warning disable CS0618 // Type or member is obsolete - case RuntimeMoniker.NetCoreApp50: -#pragma warning restore CS0618 // Type or member is obsolete - case RuntimeMoniker.Net50: - case RuntimeMoniker.Net60: - case RuntimeMoniker.Net70: - case RuntimeMoniker.Net80: - case RuntimeMoniker.Net90: - return baseJob - .WithRuntime(runtimeMoniker.GetRuntime()) - .WithToolchain(CsProjCoreToolchain.From(new NetCoreAppSettings(runtimeId, null, runtimeId, options.CliPath?.FullName, options.RestorePath?.FullName))); - - case RuntimeMoniker.Mono: - return baseJob.WithRuntime(new MonoRuntime("Mono", options.MonoPath?.FullName)); - - case RuntimeMoniker.NativeAot60: - return CreateAotJob(baseJob, options, runtimeMoniker, "6.0.0-*", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json"); - - case RuntimeMoniker.NativeAot70: - return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json"); - - case RuntimeMoniker.NativeAot80: - return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json"); - - case RuntimeMoniker.NativeAot90: - return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json"); - - case RuntimeMoniker.Wasm: - return MakeWasmJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net5.0", runtimeMoniker); - - case RuntimeMoniker.WasmNet50: - return MakeWasmJob(baseJob, options, "net5.0", runtimeMoniker); - - case RuntimeMoniker.WasmNet60: - return MakeWasmJob(baseJob, options, "net6.0", runtimeMoniker); - - case RuntimeMoniker.WasmNet70: - return MakeWasmJob(baseJob, options, "net7.0", runtimeMoniker); - - case RuntimeMoniker.WasmNet80: - return MakeWasmJob(baseJob, options, "net8.0", runtimeMoniker); - - case RuntimeMoniker.WasmNet90: - return MakeWasmJob(baseJob, options, "net9.0", runtimeMoniker); - - case RuntimeMoniker.MonoAOTLLVM: - return MakeMonoAOTLLVMJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net6.0"); - - case RuntimeMoniker.MonoAOTLLVMNet60: - return MakeMonoAOTLLVMJob(baseJob, options, "net6.0"); - - case RuntimeMoniker.MonoAOTLLVMNet70: - return MakeMonoAOTLLVMJob(baseJob, options, "net7.0"); - - case RuntimeMoniker.MonoAOTLLVMNet80: - return MakeMonoAOTLLVMJob(baseJob, options, "net8.0"); - - case RuntimeMoniker.MonoAOTLLVMNet90: - return MakeMonoAOTLLVMJob(baseJob, options, "net9.0"); - - case RuntimeMoniker.Mono60: - return MakeMonoJob(baseJob, options, MonoRuntime.Mono60); - - case RuntimeMoniker.Mono70: - return MakeMonoJob(baseJob, options, MonoRuntime.Mono70); - - case RuntimeMoniker.Mono80: - return MakeMonoJob(baseJob, options, MonoRuntime.Mono80); - - case RuntimeMoniker.Mono90: - return MakeMonoJob(baseJob, options, MonoRuntime.Mono90); - - default: - throw new NotSupportedException($"Runtime {runtimeId} is not supported"); - } - } - - private static Job CreateAotJob(Job baseJob, CommandLineOptions options, RuntimeMoniker runtimeMoniker, string ilCompilerVersion, string nuGetFeedUrl) - { - var builder = NativeAotToolchain.CreateBuilder(); - - if (options.CliPath != null) - builder.DotNetCli(options.CliPath.FullName); - if (options.RestorePath != null) - builder.PackagesRestorePath(options.RestorePath.FullName); - - if (options.IlcPackages != null) - builder.UseLocalBuild(options.IlcPackages); - else if (!string.IsNullOrEmpty(options.ILCompilerVersion)) - builder.UseNuGet(options.ILCompilerVersion, nuGetFeedUrl); - else - builder.UseNuGet(ilCompilerVersion, nuGetFeedUrl); - - var runtime = runtimeMoniker.GetRuntime(); - builder.TargetFrameworkMoniker(runtime.MsBuildMoniker); - - return baseJob.WithRuntime(runtime).WithToolchain(builder.ToToolchain()); - } - - private static Job MakeMonoJob(Job baseJob, CommandLineOptions options, MonoRuntime runtime) - { - return baseJob - .WithRuntime(runtime) - .WithToolchain(MonoToolchain.From( - new NetCoreAppSettings( - targetFrameworkMoniker: runtime.MsBuildMoniker, - runtimeFrameworkVersion: null, - name: runtime.Name, - customDotNetCliPath: options.CliPath?.FullName, - packagesPath: options.RestorePath?.FullName))); - } - - private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, string msBuildMoniker) - { - var monoAotLLVMRuntime = new MonoAotLLVMRuntime(aotCompilerPath: options.AOTCompilerPath, aotCompilerMode: options.AOTCompilerMode, msBuildMoniker: msBuildMoniker); - - var toolChain = MonoAotLLVMToolChain.From( - new NetCoreAppSettings( - targetFrameworkMoniker: monoAotLLVMRuntime.MsBuildMoniker, - runtimeFrameworkVersion: null, - name: monoAotLLVMRuntime.Name, - customDotNetCliPath: options.CliPath?.FullName, - packagesPath: options.RestorePath?.FullName, - customRuntimePack: options.CustomRuntimePack, - aotCompilerPath: options.AOTCompilerPath.ToString(), - aotCompilerMode: options.AOTCompilerMode)); - - return baseJob.WithRuntime(monoAotLLVMRuntime).WithToolchain(toolChain); - } - - private static Job MakeWasmJob(Job baseJob, CommandLineOptions options, string msBuildMoniker, RuntimeMoniker moniker) - { - bool wasmAot = options.AOTCompilerMode == MonoAotCompilerMode.wasm; - - var wasmRuntime = new WasmRuntime( - msBuildMoniker: msBuildMoniker, - javaScriptEngine: options.WasmJavascriptEngine?.FullName ?? "v8", - javaScriptEngineArguments: options.WasmJavaScriptEngineArguments, - aot: wasmAot, - wasmDataDir: options.WasmDataDirectory?.FullName, - moniker: moniker); - - var toolChain = WasmToolchain.From(new NetCoreAppSettings( - targetFrameworkMoniker: wasmRuntime.MsBuildMoniker, - runtimeFrameworkVersion: null, - name: wasmRuntime.Name, - customDotNetCliPath: options.CliPath?.FullName, - packagesPath: options.RestorePath?.FullName, - customRuntimePack: options.CustomRuntimePack, - aotCompilerMode: options.AOTCompilerMode)); - - return baseJob.WithRuntime(wasmRuntime).WithToolchain(toolChain); - } - - private static IEnumerable GetFilters(CommandLineOptions options) - { - if (options.Filters.Any()) - yield return new GlobFilter(options.Filters.ToArray()); - if (options.AllCategories.Any()) - yield return new AllCategoriesFilter(options.AllCategories.ToArray()); - if (options.AnyCategories.Any()) - yield return new AnyCategoriesFilter(options.AnyCategories.ToArray()); - if (options.AttributeNames.Any()) - yield return new AttributesFilter(options.AttributeNames.ToArray()); - } - - private static int GetMaximumDisplayWidth() - { - try - { - return Console.WindowWidth; - } - catch (IOException) - { - return MinimumDisplayWidth; - } - } - - private static Job CreateCoreRunJob(Job baseJob, CommandLineOptions options, FileInfo coreRunPath) - => baseJob - .WithToolchain(new CoreRunToolchain( - coreRunPath, - createCopy: true, - targetFrameworkMoniker: - RuntimeInformation.IsNetCore - ? RuntimeInformation.GetCurrentRuntime().MsBuildMoniker - : CoreRuntime.Latest.MsBuildMoniker, // use most recent tfm, as the toolchain is being used only by dotnet/runtime contributors - customDotNetCliPath: options.CliPath, - restorePath: options.RestorePath, - displayName: GetCoreRunToolchainDisplayName(options.CoreRunPaths, coreRunPath))); - - private static Job CreateCoreJobWithCli(Job baseJob, CommandLineOptions options) - => baseJob - .WithToolchain(CsProjCoreToolchain.From( - new NetCoreAppSettings( - targetFrameworkMoniker: RuntimeInformation.GetCurrentRuntime().MsBuildMoniker, - customDotNetCliPath: options.CliPath?.FullName, - runtimeFrameworkVersion: null, - name: RuntimeInformation.GetCurrentRuntime().Name, - packagesPath: options.RestorePath?.FullName))); - - /// - /// we have a limited amount of space when printing the output to the console, so we try to keep things small and simple - /// - /// for following paths: - /// C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root\CoreRun.exe - /// C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root_beforeMyChanges\CoreRun.exe - /// - /// we get: - /// - /// \Core_Root\CoreRun.exe - /// \Core_Root_beforeMyChanges\CoreRun.exe - /// - private static string GetCoreRunToolchainDisplayName(IReadOnlyList paths, FileInfo coreRunPath) - { - if (paths.Count <= 1) - return "CoreRun"; - - int commonLongestPrefixIndex = paths[0].FullName.Length; - for (int i = 1; i < paths.Count; i++) - { - commonLongestPrefixIndex = Math.Min(commonLongestPrefixIndex, paths[i].FullName.Length); - for (int j = 0; j < commonLongestPrefixIndex; j++) - if (paths[i].FullName[j] != paths[0].FullName[j]) - { - commonLongestPrefixIndex = j; - break; - } - } - - if (commonLongestPrefixIndex <= 1) - return coreRunPath.FullName; - - var lastCommonDirectorySeparatorIndex = coreRunPath.FullName.LastIndexOf(Path.DirectorySeparatorChar, commonLongestPrefixIndex - 1); - - return coreRunPath.FullName.Substring(lastCommonDirectorySeparatorIndex); - } - - private static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker) - { - int index = runtime.IndexOf('-'); - - return index < 0 - ? Enum.TryParse(runtime.Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker) - : Enum.TryParse(runtime.Substring(0, index).Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker); - } - } +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Text; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Exporters.Csv; +using BenchmarkDotNet.Exporters.Json; +using BenchmarkDotNet.Exporters.Xml; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Filters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Toolchains.CoreRun; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using BenchmarkDotNet.Toolchains.MonoAotLLVM; +using BenchmarkDotNet.Toolchains.MonoWasm; +using BenchmarkDotNet.Toolchains.NativeAot; +using CommandLine; +using Perfolizer.Horology; +using Perfolizer.Mathematics.OutlierDetection; +using BenchmarkDotNet.Toolchains.Mono; +using Perfolizer.Metrology; + +namespace BenchmarkDotNet.ConsoleArguments +{ + public static class ConfigParser + { + private const int MinimumDisplayWidth = 80; + private const char EnvVarKeyValueSeparator = ':'; + + private static readonly IReadOnlyDictionary AvailableJobs = new Dictionary(StringComparer.InvariantCultureIgnoreCase) + { + { "default", Job.Default }, + { "dry", Job.Dry }, + { "short", Job.ShortRun }, + { "medium", Job.MediumRun }, + { "long", Job.LongRun }, + { "verylong", Job.VeryLongRun } + }; + + [SuppressMessage("ReSharper", "StringLiteralTypo")] + [SuppressMessage("ReSharper", "CoVariantArrayConversion")] + private static readonly IReadOnlyDictionary AvailableExporters = + new Dictionary(StringComparer.InvariantCultureIgnoreCase) + { + { "csv", new[] { CsvExporter.Default } }, + { "csvmeasurements", new[] { CsvMeasurementsExporter.Default } }, + { "html", new[] { HtmlExporter.Default } }, + { "markdown", new[] { MarkdownExporter.Default } }, + { "atlassian", new[] { MarkdownExporter.Atlassian } }, + { "stackoverflow", new[] { MarkdownExporter.StackOverflow } }, + { "github", new[] { MarkdownExporter.GitHub } }, + { "plain", new[] { PlainExporter.Default } }, + { "rplot", new[] { CsvMeasurementsExporter.Default, RPlotExporter.Default } }, // R Plots depends on having the full measurements available + { "json", new[] { JsonExporter.Default } }, + { "briefjson", new[] { JsonExporter.Brief } }, + { "fulljson", new[] { JsonExporter.Full } }, + { "asciidoc", new[] { AsciiDocExporter.Default } }, + { "xml", new[] { XmlExporter.Default } }, + { "briefxml", new[] { XmlExporter.Brief } }, + { "fullxml", new[] { XmlExporter.Full } } + }; + + public static (bool isSuccess, IConfig config, CommandLineOptions options) Parse(string[] args, ILogger logger, IConfig? globalConfig = null) + { + (bool isSuccess, IConfig config, CommandLineOptions options) result = default; + + var (expandSuccess, expandedArgs) = ExpandResponseFile(args, logger); + if (!expandSuccess) + { + return (false, default, default); + } + + args = expandedArgs; + using (var parser = CreateParser(logger)) + { + parser + .ParseArguments(args) + .WithParsed(options => result = Validate(options, logger) ? (true, CreateConfig(options, globalConfig, args), options) : (false, default, default)) + .WithNotParsed(errors => result = (false, default, default)); + } + + return result; + } + + private static (bool Success, string[] ExpandedTokens) ExpandResponseFile(string[] args, ILogger logger) + { + List result = new(); + foreach (var arg in args) + { + if (arg.StartsWith("@")) + { + var fileName = arg.Substring(1); + try + { + if (File.Exists(fileName)) + { + var lines = File.ReadAllLines(fileName); + foreach (var line in lines) + { + result.AddRange(ConsumeTokens(line)); + } + } + else + { + logger.WriteLineError($"Response file {fileName} does not exists."); + return (false, Array.Empty()); + } + } + catch (Exception ex) + { + logger.WriteLineError($"Failed to parse RSP file: {fileName}, {ex.Message}"); + return (false, Array.Empty()); + } + } + else + { + result.Add(arg); + } + } + + return (true, result.ToArray()); + } + + private static IEnumerable ConsumeTokens(string line) + { + bool insideQuotes = false; + var token = new StringBuilder(); + for (int i = 0; i < line.Length; i++) + { + char currentChar = line[i]; + if (currentChar == ' ' && !insideQuotes) + { + if (token.Length > 0) + { + yield return GetToken(); + token = new StringBuilder(); + } + + continue; + } + + if (currentChar == '"') + { + insideQuotes = !insideQuotes; + continue; + } + + if (currentChar == '\\' && insideQuotes) + { + if (line[i + 1] == '"') + { + insideQuotes = false; + i++; + continue; + } + + if (line[i + 1] == '\\') + { + token.Append('\\'); + i++; + continue; + } + } + + token.Append(currentChar); + } + + if (token.Length > 0) + { + yield return GetToken(); + } + + string GetToken() + { + var result = token.ToString(); + if (result.Contains(' ')) + { + // Workaround for CommandLine library issue with parsing these kind of args. + return " " + result; + } + + return result; + } + } + + internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Action updater) + { + (bool isSuccess, CommandLineOptions options) result = default; + + ILogger logger = NullLogger.Instance; + using (var parser = CreateParser(logger)) + { + parser + .ParseArguments(args) + .WithParsed(options => result = Validate(options, logger) ? (true, options) : (false, default)) + .WithNotParsed(errors => result = (false, default)); + + if (!result.isSuccess) + { + updatedArgs = null; + return false; + } + + updater(result.options); + + updatedArgs = parser.FormatCommandLine(result.options, settings => settings.SkipDefault = true).Split(); + return true; + } + } + + private static Parser CreateParser(ILogger logger) + => new Parser(settings => + { + settings.CaseInsensitiveEnumValues = true; + settings.CaseSensitive = false; + settings.EnableDashDash = true; + settings.IgnoreUnknownArguments = false; + settings.HelpWriter = new LoggerWrapper(logger); + settings.MaximumDisplayWidth = Math.Max(MinimumDisplayWidth, GetMaximumDisplayWidth()); + }); + + private static bool Validate(CommandLineOptions options, ILogger logger) + { + if (!AvailableJobs.ContainsKey(options.BaseJob)) + { + logger.WriteLineError($"The provided base job \"{options.BaseJob}\" is invalid. Available options are: {string.Join(", ", AvailableJobs.Keys)}."); + return false; + } + + foreach (string runtime in options.Runtimes) + { + if (!TryParse(runtime, out RuntimeMoniker runtimeMoniker)) + { + logger.WriteLineError($"The provided runtime \"{runtime}\" is invalid. Available options are: {string.Join(", ", Enum.GetNames(typeof(RuntimeMoniker)).Select(name => name.ToLower()))}."); + return false; + } + else if (runtimeMoniker == RuntimeMoniker.MonoAOTLLVM && (options.AOTCompilerPath == null || options.AOTCompilerPath.IsNotNullButDoesNotExist())) + { + logger.WriteLineError($"The provided {nameof(options.AOTCompilerPath)} \"{options.AOTCompilerPath}\" does NOT exist. It MUST be provided."); + } + } + + foreach (string exporter in options.Exporters) + if (!AvailableExporters.ContainsKey(exporter)) + { + logger.WriteLineError($"The provided exporter \"{exporter}\" is invalid. Available options are: {string.Join(", ", AvailableExporters.Keys)}."); + return false; + } + + if (options.CliPath.IsNotNullButDoesNotExist()) + { + logger.WriteLineError($"The provided {nameof(options.CliPath)} \"{options.CliPath}\" does NOT exist."); + return false; + } + + foreach (var coreRunPath in options.CoreRunPaths) + if (coreRunPath.IsNotNullButDoesNotExist()) + { + if (Directory.Exists(coreRunPath.FullName)) + { + logger.WriteLineError($"The provided path to CoreRun: \"{coreRunPath}\" exists but it's a directory, not an executable. You need to include CoreRun.exe (corerun on Unix) in the path."); + } + else + { + logger.WriteLineError($"The provided path to CoreRun: \"{coreRunPath}\" does NOT exist."); + } + + return false; + } + + if (options.MonoPath.IsNotNullButDoesNotExist()) + { + logger.WriteLineError($"The provided {nameof(options.MonoPath)} \"{options.MonoPath}\" does NOT exist."); + return false; + } + + if (options.WasmJavascriptEngine.IsNotNullButDoesNotExist()) + { + logger.WriteLineError($"The provided {nameof(options.WasmJavascriptEngine)} \"{options.WasmJavascriptEngine}\" does NOT exist."); + return false; + } + + if (options.IlcPackages.IsNotNullButDoesNotExist()) + { + logger.WriteLineError($"The provided {nameof(options.IlcPackages)} \"{options.IlcPackages}\" does NOT exist."); + return false; + } + + if (options.HardwareCounters.Count() > 3) + { + logger.WriteLineError("You can't use more than 3 HardwareCounters at the same time."); + return false; + } + + foreach (var counterName in options.HardwareCounters) + if (!Enum.TryParse(counterName, ignoreCase: true, out HardwareCounter _)) + { + logger.WriteLineError($"The provided hardware counter \"{counterName}\" is invalid. Available options are: {string.Join("+", Enum.GetNames(typeof(HardwareCounter)))}."); + return false; + } + + if (!string.IsNullOrEmpty(options.StatisticalTestThreshold) && !Threshold.TryParse(options.StatisticalTestThreshold, out _)) + { + logger.WriteLineError("Invalid Threshold for Statistical Test. Use --help to see examples."); + return false; + } + + if (options.EnvironmentVariables.Any(envVar => envVar.IndexOf(EnvVarKeyValueSeparator) <= 0)) + { + logger.WriteLineError($"Environment variable value must be separated from the key using '{EnvVarKeyValueSeparator}'. Use --help to see examples."); + return false; + } + + return true; + } + + private static IConfig CreateConfig(CommandLineOptions options, IConfig globalConfig, string[] args) + { + var config = new ManualConfig(); + + var baseJob = GetBaseJob(options, globalConfig); + var expanded = Expand(baseJob.UnfreezeCopy(), options, args).ToArray(); // UnfreezeCopy ensures that each of the expanded jobs will have it's own ID + if (expanded.Length > 1) + expanded[0] = expanded[0].AsBaseline(); // if the user provides multiple jobs, then the first one should be a baseline + config.AddJob(expanded); + if (config.GetJobs().IsEmpty() && baseJob != Job.Default) + config.AddJob(baseJob); + + config.AddExporter(options.Exporters.SelectMany(exporter => AvailableExporters[exporter]).ToArray()); + + config.AddHardwareCounters(options.HardwareCounters + .Select(counterName => (HardwareCounter)Enum.Parse(typeof(HardwareCounter), counterName, ignoreCase: true)) + .ToArray()); + + if (options.UseMemoryDiagnoser) + config.AddDiagnoser(MemoryDiagnoser.Default); + if (options.UseThreadingDiagnoser) + config.AddDiagnoser(ThreadingDiagnoser.Default); + if (options.UseExceptionDiagnoser) + config.AddDiagnoser(ExceptionDiagnoser.Default); + if (options.UseDisassemblyDiagnoser) + config.AddDiagnoser(new DisassemblyDiagnoser(new DisassemblyDiagnoserConfig( + maxDepth: options.DisassemblerRecursiveDepth, + filters: options.DisassemblerFilters.ToArray(), + exportDiff: options.DisassemblerDiff))); + if (!string.IsNullOrEmpty(options.Profiler)) + config.AddDiagnoser(DiagnosersLoader.GetImplementation(profiler => profiler.ShortName.EqualsWithIgnoreCase(options.Profiler))); + + if (options.DisplayAllStatistics) + config.AddColumn(StatisticColumn.AllStatistics); + if (!string.IsNullOrEmpty(options.StatisticalTestThreshold) && Threshold.TryParse(options.StatisticalTestThreshold, out var threshold)) + config.AddColumn(new StatisticalTestColumn(threshold)); + + if (options.ArtifactsDirectory != null) + config.ArtifactsPath = options.ArtifactsDirectory.FullName; + + var filters = GetFilters(options).ToArray(); + if (filters.Length > 1) + config.AddFilter(new UnionFilter(filters)); + else + config.AddFilter(filters); + + config.HideColumns(options.HiddenColumns.ToArray()); + + config.WithOption(ConfigOptions.JoinSummary, options.Join); + config.WithOption(ConfigOptions.KeepBenchmarkFiles, options.KeepBenchmarkFiles); + config.WithOption(ConfigOptions.DontOverwriteResults, options.DontOverwriteResults); + config.WithOption(ConfigOptions.StopOnFirstError, options.StopOnFirstError); + config.WithOption(ConfigOptions.DisableLogFile, options.DisableLogFile); + config.WithOption(ConfigOptions.LogBuildOutput, options.LogBuildOutput); + config.WithOption(ConfigOptions.GenerateMSBuildBinLog, options.GenerateMSBuildBinLog); + config.WithOption(ConfigOptions.ApplesToApples, options.ApplesToApples); + config.WithOption(ConfigOptions.Resume, options.Resume); + + if (config.Options.IsSet(ConfigOptions.GenerateMSBuildBinLog)) + config.Options |= ConfigOptions.KeepBenchmarkFiles; + + if (options.MaxParameterColumnWidth.HasValue) + config.WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(options.MaxParameterColumnWidth.Value)); + + if (options.TimeOutInSeconds.HasValue) + config.WithBuildTimeout(TimeSpan.FromSeconds(options.TimeOutInSeconds.Value)); + + if (options.WakeLock.HasValue) + config.WithWakeLock(options.WakeLock.Value); + + return config; + } + + private static Job GetBaseJob(CommandLineOptions options, IConfig globalConfig) + { + var baseJob = + globalConfig?.GetJobs().SingleOrDefault(job => job.Meta.IsDefault) // global config might define single custom Default job + ?? AvailableJobs[options.BaseJob.ToLowerInvariant()]; + + if (baseJob != Job.Dry && options.Outliers != OutlierMode.RemoveUpper) + baseJob = baseJob.WithOutlierMode(options.Outliers); + + if (options.Affinity.HasValue) + baseJob = baseJob.WithAffinity((IntPtr)options.Affinity.Value); + + if (options.LaunchCount.HasValue) + baseJob = baseJob.WithLaunchCount(options.LaunchCount.Value); + if (options.WarmupIterationCount.HasValue) + baseJob = baseJob.WithWarmupCount(options.WarmupIterationCount.Value); + if (options.MinWarmupIterationCount.HasValue) + baseJob = baseJob.WithMinWarmupCount(options.MinWarmupIterationCount.Value); + if (options.MaxWarmupIterationCount.HasValue) + baseJob = baseJob.WithMaxWarmupCount(options.MaxWarmupIterationCount.Value); + if (options.IterationTimeInMilliseconds.HasValue) + baseJob = baseJob.WithIterationTime(TimeInterval.FromMilliseconds(options.IterationTimeInMilliseconds.Value)); + if (options.IterationCount.HasValue) + baseJob = baseJob.WithIterationCount(options.IterationCount.Value); + if (options.MinIterationCount.HasValue) + baseJob = baseJob.WithMinIterationCount(options.MinIterationCount.Value); + if (options.MaxIterationCount.HasValue) + baseJob = baseJob.WithMaxIterationCount(options.MaxIterationCount.Value); + if (options.InvocationCount.HasValue) + baseJob = baseJob.WithInvocationCount(options.InvocationCount.Value); + if (options.UnrollFactor.HasValue) + baseJob = baseJob.WithUnrollFactor(options.UnrollFactor.Value); + if (options.RunStrategy.HasValue) + baseJob = baseJob.WithStrategy(options.RunStrategy.Value); + if (options.Platform.HasValue) + baseJob = baseJob.WithPlatform(options.Platform.Value); + if (options.RunOncePerIteration) + baseJob = baseJob.RunOncePerIteration(); + if (options.MemoryRandomization) + baseJob = baseJob.WithMemoryRandomization(); + if (options.NoForcedGCs) + baseJob = baseJob.WithGcForce(false); + if (options.NoEvaluationOverhead) + baseJob = baseJob.WithEvaluateOverhead(false); + + if (options.EnvironmentVariables.Any()) + { + baseJob = baseJob.WithEnvironmentVariables(options.EnvironmentVariables.Select(text => + { + var separated = text.Split(new[] { EnvVarKeyValueSeparator }, 2); + return new EnvironmentVariable(separated[0], separated[1]); + }).ToArray()); + } + + if (AvailableJobs.Values.Contains(baseJob)) // no custom settings + return baseJob; + + return baseJob + .AsDefault(false) // after applying all settings from console args the base job is not default anymore + .AsMutator(); // we mark it as mutator so it will be applied to other jobs defined via attributes and merged later in GetRunnableJobs method + } + + private static IEnumerable Expand(Job baseJob, CommandLineOptions options, string[] args) + { + if (options.RunInProcess) + { + yield return baseJob.WithToolchain(InProcessEmitToolchain.Instance); + } + else if (!string.IsNullOrEmpty(options.ClrVersion)) + { + yield return baseJob.WithRuntime(ClrRuntime.CreateForLocalFullNetFrameworkBuild(options.ClrVersion)); // local builds of .NET Runtime + } + else if (options.CliPath != null && options.Runtimes.IsEmpty() && options.CoreRunPaths.IsEmpty()) + { + yield return CreateCoreJobWithCli(baseJob, options); + } + else + { + // in case both --runtimes and --corerun are specified, the first one is returned first and becomes a baseline job + string first = args.FirstOrDefault(arg => + arg.Equals("--runtimes", StringComparison.OrdinalIgnoreCase) + || arg.Equals("-r", StringComparison.OrdinalIgnoreCase) + + || arg.Equals("--corerun", StringComparison.OrdinalIgnoreCase)); + + if (first is null || first.Equals("--corerun", StringComparison.OrdinalIgnoreCase)) + { + foreach (var coreRunPath in options.CoreRunPaths) + yield return CreateCoreRunJob(baseJob, options, coreRunPath); // local dotnet/runtime builds + + foreach (string runtime in options.Runtimes) // known runtimes + yield return CreateJobForGivenRuntime(baseJob, runtime, options); + } + else + { + foreach (string runtime in options.Runtimes) // known runtimes + yield return CreateJobForGivenRuntime(baseJob, runtime, options); + + foreach (var coreRunPath in options.CoreRunPaths) + yield return CreateCoreRunJob(baseJob, options, coreRunPath); // local dotnet/runtime builds + } + } + } + + private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, CommandLineOptions options) + { + if (!TryParse(runtimeId, out RuntimeMoniker runtimeMoniker)) + { + throw new InvalidOperationException("Impossible, already validated by the Validate method"); + } + + switch (runtimeMoniker) + { + case RuntimeMoniker.Net461: + case RuntimeMoniker.Net462: + case RuntimeMoniker.Net47: + case RuntimeMoniker.Net471: + case RuntimeMoniker.Net472: + case RuntimeMoniker.Net48: + case RuntimeMoniker.Net481: + return baseJob + .WithRuntime(runtimeMoniker.GetRuntime()) + .WithToolchain(CsProjClassicNetToolchain.From(runtimeId, options.RestorePath?.FullName, options.CliPath?.FullName)); + + case RuntimeMoniker.NetCoreApp20: + case RuntimeMoniker.NetCoreApp21: + case RuntimeMoniker.NetCoreApp22: + case RuntimeMoniker.NetCoreApp30: + case RuntimeMoniker.NetCoreApp31: +#pragma warning disable CS0618 // Type or member is obsolete + case RuntimeMoniker.NetCoreApp50: +#pragma warning restore CS0618 // Type or member is obsolete + case RuntimeMoniker.Net50: + case RuntimeMoniker.Net60: + case RuntimeMoniker.Net70: + case RuntimeMoniker.Net80: + case RuntimeMoniker.Net90: + case RuntimeMoniker.Net10_0: + return baseJob + .WithRuntime(runtimeMoniker.GetRuntime()) + .WithToolchain(CsProjCoreToolchain.From(new NetCoreAppSettings(runtimeId, null, runtimeId, options.CliPath?.FullName, options.RestorePath?.FullName))); + + case RuntimeMoniker.Mono: + return baseJob.WithRuntime(new MonoRuntime("Mono", options.MonoPath?.FullName)); + + case RuntimeMoniker.NativeAot60: + return CreateAotJob(baseJob, options, runtimeMoniker, "6.0.0-*", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json"); + + case RuntimeMoniker.NativeAot70: + return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://api.nuget.org/v3/index.json"); + + case RuntimeMoniker.NativeAot80: + return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://api.nuget.org/v3/index.json"); + + case RuntimeMoniker.NativeAot90: + return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json"); + + case RuntimeMoniker.NativeAot10_0: + return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json"); + + case RuntimeMoniker.Wasm: + return MakeWasmJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net5.0", runtimeMoniker); + + case RuntimeMoniker.WasmNet50: + return MakeWasmJob(baseJob, options, "net5.0", runtimeMoniker); + + case RuntimeMoniker.WasmNet60: + return MakeWasmJob(baseJob, options, "net6.0", runtimeMoniker); + + case RuntimeMoniker.WasmNet70: + return MakeWasmJob(baseJob, options, "net7.0", runtimeMoniker); + + case RuntimeMoniker.WasmNet80: + return MakeWasmJob(baseJob, options, "net8.0", runtimeMoniker); + + case RuntimeMoniker.WasmNet90: + return MakeWasmJob(baseJob, options, "net9.0", runtimeMoniker); + + case RuntimeMoniker.WasmNet10_0: + return MakeWasmJob(baseJob, options, "net10.0", runtimeMoniker); + + case RuntimeMoniker.MonoAOTLLVM: + return MakeMonoAOTLLVMJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net6.0", runtimeMoniker); + + case RuntimeMoniker.MonoAOTLLVMNet60: + return MakeMonoAOTLLVMJob(baseJob, options, "net6.0", runtimeMoniker); + + case RuntimeMoniker.MonoAOTLLVMNet70: + return MakeMonoAOTLLVMJob(baseJob, options, "net7.0", runtimeMoniker); + + case RuntimeMoniker.MonoAOTLLVMNet80: + return MakeMonoAOTLLVMJob(baseJob, options, "net8.0", runtimeMoniker); + + case RuntimeMoniker.MonoAOTLLVMNet90: + return MakeMonoAOTLLVMJob(baseJob, options, "net9.0", runtimeMoniker); + + case RuntimeMoniker.MonoAOTLLVMNet10_0: + return MakeMonoAOTLLVMJob(baseJob, options, "net10.0", runtimeMoniker); + + case RuntimeMoniker.Mono60: + return MakeMonoJob(baseJob, options, MonoRuntime.Mono60); + + case RuntimeMoniker.Mono70: + return MakeMonoJob(baseJob, options, MonoRuntime.Mono70); + + case RuntimeMoniker.Mono80: + return MakeMonoJob(baseJob, options, MonoRuntime.Mono80); + + case RuntimeMoniker.Mono90: + return MakeMonoJob(baseJob, options, MonoRuntime.Mono90); + + case RuntimeMoniker.Mono10_0: + return MakeMonoJob(baseJob, options, MonoRuntime.Mono10_0); + + default: + throw new NotSupportedException($"Runtime {runtimeId} is not supported"); + } + } + + private static Job CreateAotJob(Job baseJob, CommandLineOptions options, RuntimeMoniker runtimeMoniker, string ilCompilerVersion, string nuGetFeedUrl) + { + var builder = NativeAotToolchain.CreateBuilder(); + + if (options.CliPath != null) + builder.DotNetCli(options.CliPath.FullName); + if (options.RestorePath != null) + builder.PackagesRestorePath(options.RestorePath.FullName); + + if (options.IlcPackages != null) + builder.UseLocalBuild(options.IlcPackages); + else if (!string.IsNullOrEmpty(options.ILCompilerVersion)) + builder.UseNuGet(options.ILCompilerVersion, nuGetFeedUrl); + else + builder.UseNuGet(ilCompilerVersion, nuGetFeedUrl); + + var runtime = runtimeMoniker.GetRuntime(); + builder.TargetFrameworkMoniker(runtime.MsBuildMoniker); + + return baseJob.WithRuntime(runtime).WithToolchain(builder.ToToolchain()); + } + + private static Job MakeMonoJob(Job baseJob, CommandLineOptions options, MonoRuntime runtime) + { + return baseJob + .WithRuntime(runtime) + .WithToolchain(MonoToolchain.From( + new NetCoreAppSettings( + targetFrameworkMoniker: runtime.MsBuildMoniker, + runtimeFrameworkVersion: null, + name: runtime.Name, + customDotNetCliPath: options.CliPath?.FullName, + packagesPath: options.RestorePath?.FullName))); + } + + private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, string msBuildMoniker, RuntimeMoniker moniker) + { + var monoAotLLVMRuntime = new MonoAotLLVMRuntime(aotCompilerPath: options.AOTCompilerPath, aotCompilerMode: options.AOTCompilerMode, msBuildMoniker: msBuildMoniker, moniker: moniker); + + var toolChain = MonoAotLLVMToolChain.From( + new NetCoreAppSettings( + targetFrameworkMoniker: monoAotLLVMRuntime.MsBuildMoniker, + runtimeFrameworkVersion: null, + name: monoAotLLVMRuntime.Name, + customDotNetCliPath: options.CliPath?.FullName, + packagesPath: options.RestorePath?.FullName, + customRuntimePack: options.CustomRuntimePack, + aotCompilerPath: options.AOTCompilerPath.ToString(), + aotCompilerMode: options.AOTCompilerMode)); + + return baseJob.WithRuntime(monoAotLLVMRuntime).WithToolchain(toolChain); + } + + private static Job MakeWasmJob(Job baseJob, CommandLineOptions options, string msBuildMoniker, RuntimeMoniker moniker) + { + bool wasmAot = options.AOTCompilerMode == MonoAotCompilerMode.wasm; + + var wasmRuntime = new WasmRuntime( + msBuildMoniker: msBuildMoniker, + javaScriptEngine: options.WasmJavascriptEngine?.FullName ?? "v8", + javaScriptEngineArguments: options.WasmJavaScriptEngineArguments, + aot: wasmAot, + wasmDataDir: options.WasmDataDirectory?.FullName, + moniker: moniker); + + var toolChain = WasmToolchain.From(new NetCoreAppSettings( + targetFrameworkMoniker: wasmRuntime.MsBuildMoniker, + runtimeFrameworkVersion: null, + name: wasmRuntime.Name, + customDotNetCliPath: options.CliPath?.FullName, + packagesPath: options.RestorePath?.FullName, + customRuntimePack: options.CustomRuntimePack, + aotCompilerMode: options.AOTCompilerMode)); + + return baseJob.WithRuntime(wasmRuntime).WithToolchain(toolChain); + } + + private static IEnumerable GetFilters(CommandLineOptions options) + { + if (options.Filters.Any()) + yield return new GlobFilter(options.Filters.ToArray()); + if (options.AllCategories.Any()) + yield return new AllCategoriesFilter(options.AllCategories.ToArray()); + if (options.AnyCategories.Any()) + yield return new AnyCategoriesFilter(options.AnyCategories.ToArray()); + if (options.AttributeNames.Any()) + yield return new AttributesFilter(options.AttributeNames.ToArray()); + } + + private static int GetMaximumDisplayWidth() + { + try + { + return Console.WindowWidth; + } + catch (IOException) + { + return MinimumDisplayWidth; + } + } + + private static Job CreateCoreRunJob(Job baseJob, CommandLineOptions options, FileInfo coreRunPath) + => baseJob + .WithToolchain(new CoreRunToolchain( + coreRunPath, + createCopy: true, + targetFrameworkMoniker: + RuntimeInformation.IsNetCore + ? RuntimeInformation.GetCurrentRuntime().MsBuildMoniker + : CoreRuntime.Latest.MsBuildMoniker, // use most recent tfm, as the toolchain is being used only by dotnet/runtime contributors + customDotNetCliPath: options.CliPath, + restorePath: options.RestorePath, + displayName: GetCoreRunToolchainDisplayName(options.CoreRunPaths, coreRunPath))); + + private static Job CreateCoreJobWithCli(Job baseJob, CommandLineOptions options) + => baseJob + .WithToolchain(CsProjCoreToolchain.From( + new NetCoreAppSettings( + targetFrameworkMoniker: RuntimeInformation.GetCurrentRuntime().MsBuildMoniker, + customDotNetCliPath: options.CliPath?.FullName, + runtimeFrameworkVersion: null, + name: RuntimeInformation.GetCurrentRuntime().Name, + packagesPath: options.RestorePath?.FullName))); + + /// + /// we have a limited amount of space when printing the output to the console, so we try to keep things small and simple + /// + /// for following paths: + /// C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root\CoreRun.exe + /// C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root_beforeMyChanges\CoreRun.exe + /// + /// we get: + /// + /// \Core_Root\CoreRun.exe + /// \Core_Root_beforeMyChanges\CoreRun.exe + /// + private static string GetCoreRunToolchainDisplayName(IReadOnlyList paths, FileInfo coreRunPath) + { + if (paths.Count <= 1) + return "CoreRun"; + + int commonLongestPrefixIndex = paths[0].FullName.Length; + for (int i = 1; i < paths.Count; i++) + { + commonLongestPrefixIndex = Math.Min(commonLongestPrefixIndex, paths[i].FullName.Length); + for (int j = 0; j < commonLongestPrefixIndex; j++) + if (paths[i].FullName[j] != paths[0].FullName[j]) + { + commonLongestPrefixIndex = j; + break; + } + } + + if (commonLongestPrefixIndex <= 1) + return coreRunPath.FullName; + + var lastCommonDirectorySeparatorIndex = coreRunPath.FullName.LastIndexOf(Path.DirectorySeparatorChar, commonLongestPrefixIndex - 1); + + return coreRunPath.FullName.Substring(lastCommonDirectorySeparatorIndex); + } + + internal static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker) + { + int index = runtime.IndexOf('-'); + if (index >= 0) + { + runtime = runtime.Substring(0, index); + } + + // Monikers older than Net 10 don't use any version delimiter, newer monikers use _ delimiter. + if (Enum.TryParse(runtime.Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker)) + { + return true; + } + return Enum.TryParse(runtime.Replace('.', '_'), ignoreCase: true, out runtimeMoniker); + } + } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/HardwareIntrinsics.cs b/src/BenchmarkDotNet/Detectors/Cpu/HardwareIntrinsics.cs similarity index 52% rename from src/BenchmarkDotNet/Portability/Cpu/HardwareIntrinsics.cs rename to src/BenchmarkDotNet/Detectors/Cpu/HardwareIntrinsics.cs index 68c48809b7..5b2815cc43 100644 --- a/src/BenchmarkDotNet/Portability/Cpu/HardwareIntrinsics.cs +++ b/src/BenchmarkDotNet/Detectors/Cpu/HardwareIntrinsics.cs @@ -1,45 +1,50 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Numerics; +using System.Text; using BenchmarkDotNet.Environments; -using System.Diagnostics.CodeAnalysis; #if NET6_0_OR_GREATER using System.Runtime.Intrinsics.X86; using System.Runtime.Intrinsics.Arm; #endif -namespace BenchmarkDotNet.Portability.Cpu +namespace BenchmarkDotNet.Detectors.Cpu { + // based on https://github.com/dotnet/runtime/tree/v10.0.0-rc.1.25451.107/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt internal static class HardwareIntrinsics { internal static string GetVectorSize() => Vector.IsHardwareAccelerated ? $"VectorSize={Vector.Count * 8}" : string.Empty; internal static string GetShortInfo() { - if (IsX86Avx2Supported) - return "AVX2"; - else if (IsX86AvxSupported) - return "AVX"; - else if (IsX86Sse42Supported) - return "SSE4.2"; - else if (IsX86Sse41Supported) - return "SSE4.1"; - else if (IsX86Ssse3Supported) - return "SSSE3"; - else if (IsX86Sse3Supported) - return "SSE3"; - else if (IsX86Sse2Supported) - return "SSE2"; - else if (IsX86SseSupported) - return "SSE"; - else if (IsX86BaseSupported) - return "X86Base"; - else if (IsArmAdvSimdSupported) - return "AdvSIMD"; + if (IsX86BaseSupported) + { + if (IsX86Avx512Supported) + { + return "x86-64-v4"; + } + else if (IsX86Avx2Supported) + { + return "x86-64-v3"; + } + else if (IsX86Sse42Supported) + { + return "x86-64-v2"; + } + else + { + return "x86-64-v1"; + } + } else if (IsArmBaseSupported) - return "ArmBase"; + { + return "armv8.0-a"; + } else + { return GetVectorSize(); // Runtimes prior to .NET Core 3.0 (APIs did not exist so we print non-exact Vector info) + } } internal static string GetFullInfo(Platform platform) @@ -52,30 +57,31 @@ static IEnumerable GetCurrentProcessInstructionSets(Platform platform) { case Platform.X86: case Platform.X64: - if (IsX86Avx2Supported) yield return "AVX2"; - else if (IsX86AvxSupported) yield return "AVX"; - else if (IsX86Sse42Supported) yield return "SSE4.2"; - else if (IsX86Sse41Supported) yield return "SSE4.1"; - else if (IsX86Ssse3Supported) yield return "SSSE3"; - else if (IsX86Sse3Supported) yield return "SSE3"; - else if (IsX86Sse2Supported) yield return "SSE2"; - else if (IsX86SseSupported) yield return "SSE"; - else if (IsX86BaseSupported) yield return "X86Base"; - - if (IsX86AesSupported) yield return "AES"; - if (IsX86Bmi1Supported) yield return "BMI1"; - if (IsX86Bmi2Supported) yield return "BMI2"; - if (IsX86FmaSupported) yield return "FMA"; - if (IsX86LzcntSupported) yield return "LZCNT"; - if (IsX86PclmulqdqSupported) yield return "PCLMUL"; - if (IsX86PopcntSupported) yield return "POPCNT"; + { + if (IsX86Avx10v2Supported) yield return "AVX10v2"; + if (IsX86Avx10v1Supported) + { + yield return "AVX10v1"; + yield return "AVX512 BF16+FP16"; + } + if (IsX86Avx512v3Supported) yield return "AVX512 BITALG+VBMI2+VNNI+VPOPCNTDQ"; + if (IsX86Avx512v2Supported) yield return "AVX512 IFMA+VBMI"; + if (IsX86Avx512Supported) yield return "AVX512 F+BW+CD+DQ+VL"; + if (IsX86Avx2Supported) yield return "AVX2+BMI1+BMI2+F16C+FMA+LZCNT+MOVBE"; + if (IsX86AvxSupported) yield return "AVX"; + if (IsX86Sse42Supported) yield return "SSE3+SSSE3+SSE4.1+SSE4.2+POPCNT"; + if (IsX86BaseSupported) yield return "X86Base+SSE+SSE2"; + if (IsX86AesSupported) yield return "AES+PCLMUL"; if (IsX86AvxVnniSupported) yield return "AvxVnni"; if (IsX86SerializeSupported) yield return "SERIALIZE"; - // TODO: Add MOVBE when API is added. break; + } case Platform.Arm64: - if (IsArmAdvSimdSupported) yield return "AdvSIMD"; - else if (IsArmBaseSupported) yield return "ArmBase"; + { + if (IsArmBaseSupported) + { + yield return "ArmBase+AdvSimd"; + } if (IsArmAesSupported) yield return "AES"; if (IsArmCrc32Supported) yield return "CRC32"; @@ -84,59 +90,39 @@ static IEnumerable GetCurrentProcessInstructionSets(Platform platform) if (IsArmSha1Supported) yield return "SHA1"; if (IsArmSha256Supported) yield return "SHA256"; break; + } + default: yield break; } } } +#pragma warning disable CA2252 // Some APIs require opting into preview features internal static bool IsX86BaseSupported => #if NET6_0_OR_GREATER - X86Base.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.X86Base"); -#endif - - internal static bool IsX86SseSupported => -#if NET6_0_OR_GREATER - Sse.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Sse"); -#endif - - internal static bool IsX86Sse2Supported => -#if NET6_0_OR_GREATER + X86Base.IsSupported && + Sse.IsSupported && Sse2.IsSupported; #elif NETSTANDARD + GetIsSupported("System.Runtime.Intrinsics.X86.X86Base") && + GetIsSupported("System.Runtime.Intrinsics.X86.Sse") && GetIsSupported("System.Runtime.Intrinsics.X86.Sse2"); #endif - internal static bool IsX86Sse3Supported => -#if NET6_0_OR_GREATER - Sse3.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Sse3"); -#endif - - internal static bool IsX86Ssse3Supported => -#if NET6_0_OR_GREATER - Ssse3.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Ssse3"); -#endif - - internal static bool IsX86Sse41Supported => -#if NET6_0_OR_GREATER - Sse41.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Sse41"); -#endif - internal static bool IsX86Sse42Supported => #if NET6_0_OR_GREATER - Sse42.IsSupported; + Sse3.IsSupported && + Ssse3.IsSupported && + Sse41.IsSupported && + Sse42.IsSupported && + Popcnt.IsSupported; #elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Sse42"); + GetIsSupported("System.Runtime.Intrinsics.X86.Sse3") && + GetIsSupported("System.Runtime.Intrinsics.X86.Ssse3") && + GetIsSupported("System.Runtime.Intrinsics.X86.Sse41") && + GetIsSupported("System.Runtime.Intrinsics.X86.Sse42") && + GetIsSupported("System.Runtime.Intrinsics.X86.Popcnt"); #endif internal static bool IsX86AvxSupported => @@ -148,83 +134,105 @@ static IEnumerable GetCurrentProcessInstructionSets(Platform platform) internal static bool IsX86Avx2Supported => #if NET6_0_OR_GREATER - Avx2.IsSupported; + Avx2.IsSupported && + Bmi1.IsSupported && + Bmi2.IsSupported && + Fma.IsSupported && + Lzcnt.IsSupported; #elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Avx2"); + GetIsSupported("System.Runtime.Intrinsics.X86.Avx2") && + GetIsSupported("System.Runtime.Intrinsics.X86.Bmi1") && + GetIsSupported("System.Runtime.Intrinsics.X86.Bmi2") && + GetIsSupported("System.Runtime.Intrinsics.X86.Fma") && + GetIsSupported("System.Runtime.Intrinsics.X86.Lzcnt"); #endif - internal static bool IsX86AesSupported => -#if NET6_0_OR_GREATER - System.Runtime.Intrinsics.X86.Aes.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Aes"); + internal static bool IsX86Avx512Supported => +#if NET8_0_OR_GREATER + Avx512F.IsSupported && + Avx512F.VL.IsSupported && + Avx512BW.IsSupported && + Avx512BW.VL.IsSupported && + Avx512CD.IsSupported && + Avx512CD.VL.IsSupported && + Avx512DQ.IsSupported && + Avx512DQ.VL.IsSupported; +#else + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512F") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512F+VL") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512BW") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512BW+VL") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512CD") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512CD+VL") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512DQ") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512DQ+VL"); #endif - internal static bool IsX86Bmi1Supported => -#if NET6_0_OR_GREATER - Bmi1.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Bmi1"); + internal static bool IsX86Avx512v2Supported => +#if NET8_0_OR_GREATER + Avx512Vbmi.IsSupported && + Avx512Vbmi.VL.IsSupported; +#else + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512Vbmi") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512Vbmi+VL"); #endif - internal static bool IsX86Bmi2Supported => -#if NET6_0_OR_GREATER - Bmi2.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Bmi2"); + internal static bool IsX86Avx512v3Supported => +#if NET10_0_OR_GREATER + Avx512Vbmi2.IsSupported && + Avx512Vbmi2.VL.IsSupported; +#else + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512Vbmi2") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512Vbmi2+VL"); #endif - internal static bool IsX86FmaSupported => -#if NET6_0_OR_GREATER - Fma.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Fma"); + internal static bool IsX86Avx10v1Supported => +#if NET9_0_OR_GREATER + Avx10v1.IsSupported && + Avx10v1.V512.IsSupported; +#else + GetIsSupported("System.Runtime.Intrinsics.X86.Avx10v1") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx10v1+V512"); #endif - internal static bool IsX86LzcntSupported => -#if NET6_0_OR_GREATER - Lzcnt.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Lzcnt"); + internal static bool IsX86Avx10v2Supported => +#if NET10_0_OR_GREATER + Avx10v2.IsSupported && + Avx10v2.V512.IsSupported; +#else + GetIsSupported("System.Runtime.Intrinsics.X86.Avx10v2") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx10v2+V512"); #endif - internal static bool IsX86PclmulqdqSupported => + internal static bool IsX86AesSupported => #if NET6_0_OR_GREATER + System.Runtime.Intrinsics.X86.Aes.IsSupported && Pclmulqdq.IsSupported; #elif NETSTANDARD + GetIsSupported("System.Runtime.Intrinsics.X86.Aes") && GetIsSupported("System.Runtime.Intrinsics.X86.Pclmulqdq"); #endif - internal static bool IsX86PopcntSupported => -#if NET6_0_OR_GREATER - Popcnt.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Popcnt"); -#endif - internal static bool IsX86AvxVnniSupported => #if NET6_0_OR_GREATER -#pragma warning disable CA2252 // This API requires opting into preview features AvxVnni.IsSupported; -#pragma warning restore CA2252 // This API requires opting into preview features #elif NETSTANDARD GetIsSupported("System.Runtime.Intrinsics.X86.AvxVnni"); #endif - // X86Serialize was introduced in .NET 7.0, BDN does not target it so we need to use reflection - internal static bool IsX86SerializeSupported => GetIsSupported("System.Runtime.Intrinsics.X86.X86Serialize"); - - internal static bool IsArmBaseSupported => -#if NET6_0_OR_GREATER - ArmBase.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.Arm.ArmBase"); + internal static bool IsX86SerializeSupported => +#if NET7_0_OR_GREATER + X86Serialize.IsSupported; +#else + GetIsSupported("System.Runtime.Intrinsics.X86.X86Serialize"); #endif - internal static bool IsArmAdvSimdSupported => + internal static bool IsArmBaseSupported => #if NET6_0_OR_GREATER + ArmBase.IsSupported && AdvSimd.IsSupported; #elif NETSTANDARD + GetIsSupported("System.Runtime.Intrinsics.Arm.ArmBase") && GetIsSupported("System.Runtime.Intrinsics.Arm.AdvSimd"); #endif @@ -269,6 +277,7 @@ static IEnumerable GetCurrentProcessInstructionSets(Platform platform) #elif NETSTANDARD GetIsSupported("System.Runtime.Intrinsics.Arm.Sha256"); #endif +#pragma warning restore CA2252 // Some APIs require opting into preview features private static bool GetIsSupported([DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties)] string typeName) { diff --git a/src/BenchmarkDotNet/Detectors/Cpu/ICpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/ICpuDetector.cs new file mode 100644 index 0000000000..72355b60e5 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/ICpuDetector.cs @@ -0,0 +1,12 @@ +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu; + +/// +/// Loads the for the current hardware +/// +public interface ICpuDetector +{ + bool IsApplicable(); + CpuInfo? Detect(); +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuDetector.cs new file mode 100644 index 0000000000..44973b27e9 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuDetector.cs @@ -0,0 +1,35 @@ +using System.Collections.Generic; +using BenchmarkDotNet.Helpers; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.Linux; + +/// +/// CPU information from output of the `cat /proc/cpuinfo` and `lscpu` command. +/// Linux only. +/// +internal class LinuxCpuDetector : ICpuDetector +{ + public bool IsApplicable() => OsDetector.IsLinux(); + + public CpuInfo? Detect() + { + if (!IsApplicable()) return null; + + // lscpu output respects the system locale, so we should force language invariant environment for correct parsing + var languageInvariantEnvironment = new Dictionary + { + ["LC_ALL"] = "C", + ["LANG"] = "C", + ["LANGUAGE"] = "C" + }; + + string? cpuInfo = ProcessHelper.RunAndReadOutput("cat", "/proc/cpuinfo") ?? string.Empty; + string? lscpu = ProcessHelper.RunAndReadOutput("/bin/bash", "-c \"lscpu\"", environmentVariables: languageInvariantEnvironment) ?? string.Empty; + + if (cpuInfo == string.Empty && lscpu == string.Empty) + return null; + + return LinuxCpuInfoParser.Parse(cpuInfo, lscpu); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuInfoParser.cs b/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuInfoParser.cs new file mode 100644 index 0000000000..7d79c3262f --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuInfoParser.cs @@ -0,0 +1,160 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Portability; +using Perfolizer.Horology; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.Linux; + +internal static class LinuxCpuInfoParser +{ + private static class ProcCpu + { + internal const string PhysicalId = "physical id"; + internal const string CpuCores = "cpu cores"; + internal const string ModelName = "model name"; + internal const string MaxFrequency = "max freq"; + internal const string NominalFrequencyBackup = "nominal freq"; + internal const string NominalFrequency = "cpu MHz"; + } + + private static class Lscpu + { + internal const string MaxFrequency = "CPU max MHz"; + internal const string ModelName = "Model name"; + internal const string CoresPerSocket = "Core(s) per socket"; + } + + /// Output of `cat /proc/cpuinfo` + /// Output of `lscpu` + internal static CpuInfo Parse(string cpuInfo, string lscpu) + { + var processorModelNames = new HashSet(); + var processorsToPhysicalCoreCount = new Dictionary(); + int logicalCoreCount = 0; + double maxFrequency = 0.0; + double nominalFrequency = 0.0; + + var logicalCores = SectionsHelper.ParseSections(cpuInfo, ':'); + foreach (var logicalCore in logicalCores) + { + if (logicalCore.TryGetValue(ProcCpu.PhysicalId, out string physicalId) && + logicalCore.TryGetValue(ProcCpu.CpuCores, out string cpuCoresValue) && + int.TryParse(cpuCoresValue, out int cpuCoreCount) && + cpuCoreCount > 0) + processorsToPhysicalCoreCount[physicalId] = cpuCoreCount; + + if (logicalCore.TryGetValue(ProcCpu.ModelName, out string modelName)) + { + processorModelNames.Add(modelName); + logicalCoreCount++; + } + + if (logicalCore.TryGetValue(ProcCpu.MaxFrequency, out string maxCpuFreqValue) && + Frequency.TryParseMHz(maxCpuFreqValue.Replace(',', '.'), out Frequency maxCpuFreq) + && maxCpuFreq > 0) + { + maxFrequency = Math.Max(maxFrequency, maxCpuFreq.ToMHz()); + } + + bool nominalFrequencyHasValue = logicalCore.TryGetValue(ProcCpu.NominalFrequency, out string nominalFreqValue); + bool nominalFrequencyBackupHasValue = logicalCore.TryGetValue(ProcCpu.NominalFrequencyBackup, out string nominalFreqBackupValue); + + double nominalCpuFreq = 0.0; + double nominalCpuBackupFreq = 0.0; + + if (nominalFrequencyHasValue && + double.TryParse(nominalFreqValue, out nominalCpuFreq) + && nominalCpuFreq > 0) + { + nominalCpuFreq = nominalFrequency == 0 ? nominalCpuFreq : Math.Min(nominalFrequency, nominalCpuFreq); + } + if (nominalFrequencyBackupHasValue && + double.TryParse(nominalFreqBackupValue, out nominalCpuBackupFreq) + && nominalCpuBackupFreq > 0) + { + nominalCpuBackupFreq = nominalFrequency == 0 ? nominalCpuBackupFreq : Math.Min(nominalFrequency, nominalCpuBackupFreq); + } + + if (nominalFrequencyHasValue && nominalFrequencyBackupHasValue) + { + nominalFrequency = Math.Min(nominalCpuFreq, nominalCpuBackupFreq); + } + else + { + nominalFrequency = nominalCpuFreq == 0.0 ? nominalCpuBackupFreq : nominalCpuFreq; + } + } + + int? coresPerSocket = null; + if (string.IsNullOrEmpty(lscpu) == false) + { + var lscpuParts = lscpu.Split('\n') + .Where(line => line.Contains(':')) + .SelectMany(line => line.Split([':'], 2)) + .ToList(); + for (int i = 0; i + 1 < lscpuParts.Count; i += 2) + { + string name = lscpuParts[i].Trim(); + string value = lscpuParts[i + 1].Trim(); + + if (name.EqualsWithIgnoreCase(Lscpu.MaxFrequency) && + Frequency.TryParseMHz(value.Replace(',', '.'), out Frequency maxFrequencyParsed)) // Example: `CPU max MHz: 3200,0000` + maxFrequency = Math.Max(maxFrequency, maxFrequencyParsed.ToMHz()); + + if (name.EqualsWithIgnoreCase(Lscpu.ModelName)) + processorModelNames.Add(value); + + if (name.EqualsWithIgnoreCase(Lscpu.CoresPerSocket) && + int.TryParse(value, out int coreCount)) + coresPerSocket = coreCount; + } + } + + string processorName = processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null; + int? physicalProcessorCount = processorsToPhysicalCoreCount.Count > 0 ? processorsToPhysicalCoreCount.Count : null; + int? physicalCoreCount = processorsToPhysicalCoreCount.Count > 0 ? processorsToPhysicalCoreCount.Values.Sum() : coresPerSocket; + + Frequency? maxFrequencyActual = maxFrequency > 0 && physicalProcessorCount > 0 + ? Frequency.FromMHz(maxFrequency) : null; + + Frequency? nominalFrequencyActual = nominalFrequency > 0 && physicalProcessorCount > 0 + ? Frequency.FromMHz(nominalFrequency) : null; + + if (nominalFrequencyActual is null) + { + bool nominalFrequencyInBrandString = processorModelNames.Any(x => ParseFrequencyFromBrandString(x) is not null); + + if (nominalFrequencyInBrandString) + nominalFrequencyActual = processorModelNames.Select(x => ParseFrequencyFromBrandString(x)) + .First(x => x is not null); + } + + return new CpuInfo + { + ProcessorName = processorName, + PhysicalProcessorCount = physicalProcessorCount, + PhysicalCoreCount = physicalCoreCount, + LogicalCoreCount = logicalCoreCount > 0 ? logicalCoreCount : null, + NominalFrequencyHz = nominalFrequencyActual?.Hertz.RoundToLong(), + MaxFrequencyHz = maxFrequencyActual?.Hertz.RoundToLong() + }; + } + + internal static Frequency? ParseFrequencyFromBrandString(string brandString) + { + const string pattern = "(\\d.\\d+)GHz"; + var matches = Regex.Matches(brandString, pattern, RegexOptions.IgnoreCase); + if (matches.Count > 0 && matches[0].Groups.Count > 1) + { + string match = Regex.Matches(brandString, pattern, RegexOptions.IgnoreCase)[0].Groups[1].ToString(); + return Frequency.TryParseGHz(match, out Frequency result) ? result : null; + } + + return null; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/MosCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/MosCpuDetector.cs new file mode 100644 index 0000000000..edc48ba25d --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/MosCpuDetector.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Management; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Portability; +using Perfolizer.Horology; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +internal class MosCpuDetector : ICpuDetector +{ +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatform("windows")] +#endif + public bool IsApplicable() => OsDetector.IsWindows() && + RuntimeInformation.IsFullFramework && + !RuntimeInformation.IsMono; + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatform("windows")] +#endif + public CpuInfo? Detect() + { + if (!IsApplicable()) return null; + + var processorModelNames = new HashSet(); + int physicalCoreCount = 0; + int logicalCoreCount = 0; + int processorsCount = 0; + double maxFrequency = 0; + double nominalFrequency = 0; + + using (var mosProcessor = new ManagementObjectSearcher("SELECT * FROM Win32_Processor")) + { + foreach (var moProcessor in mosProcessor.Get().Cast()) + { + string name = moProcessor[WmicCpuInfoKeyNames.Name]?.ToString(); + if (!string.IsNullOrEmpty(name)) + { + processorModelNames.Add(name); + processorsCount++; + physicalCoreCount += (int)(uint)moProcessor[WmicCpuInfoKeyNames.NumberOfCores]; + logicalCoreCount += (int)(uint)moProcessor[WmicCpuInfoKeyNames.NumberOfLogicalProcessors]; + double tempMaxFrequency = (uint)moProcessor[WmicCpuInfoKeyNames.MaxClockSpeed]; + + if (tempMaxFrequency > 0) + { + nominalFrequency = nominalFrequency == 0 ? tempMaxFrequency : Math.Min(nominalFrequency, tempMaxFrequency); + } + maxFrequency = Math.Max(maxFrequency, tempMaxFrequency); + } + } + } + + string processorName = processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null; + Frequency? maxFrequencyActual = maxFrequency > 0 && processorsCount > 0 + ? Frequency.FromMHz(maxFrequency) + : null; + Frequency? nominalFrequencyActual = nominalFrequency > 0 && processorsCount > 0 + ? Frequency.FromMHz(nominalFrequency) + : null; + + return new CpuInfo + { + ProcessorName = processorName, + PhysicalProcessorCount = processorsCount > 0 ? processorsCount : null, + PhysicalCoreCount = physicalCoreCount > 0 ? physicalCoreCount : null, + LogicalCoreCount = logicalCoreCount > 0 ? logicalCoreCount : null, + NominalFrequencyHz = nominalFrequencyActual?.Hertz.RoundToLong(), + MaxFrequencyHz = maxFrequencyActual?.Hertz.RoundToLong() + }; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/PowershellWmiCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/PowershellWmiCpuDetector.cs new file mode 100644 index 0000000000..7f61c1b8fc --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/PowershellWmiCpuDetector.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.Versioning; +using System.Text.RegularExpressions; +using BenchmarkDotNet.Helpers; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +/// +/// CPU information from output of the `wmic cpu get Name, NumberOfCores, NumberOfLogicalProcessors /Format:List` command. +/// Windows only. +/// +internal class PowershellWmiCpuDetector : ICpuDetector +{ + private readonly string windowsPowershellPath = + $"{Environment.SystemDirectory}{Path.DirectorySeparatorChar}WindowsPowerShell{Path.DirectorySeparatorChar}" + + $"v1.0{Path.DirectorySeparatorChar}powershell.exe"; + + public bool IsApplicable() => OsDetector.IsWindows(); + + #if NET6_0_OR_GREATER + [SupportedOSPlatform("windows")] + #endif + public CpuInfo? Detect() + { + if (!IsApplicable()) return null; + + const string argList = $"{WmicCpuInfoKeyNames.Name}, " + + $"{WmicCpuInfoKeyNames.NumberOfCores}, " + + $"{WmicCpuInfoKeyNames.NumberOfLogicalProcessors}, " + + $"{WmicCpuInfoKeyNames.MaxClockSpeed}"; + + string output = ProcessHelper.RunAndReadOutput(PowerShellLocator.LocateOnWindows() ?? "PowerShell", + "Get-CimInstance Win32_Processor -Property " + argList); + + if (string.IsNullOrEmpty(output)) + return null; + + return PowershellWmiCpuInfoParser.Parse(output); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/PowershellWmiCpuInfoParser.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/PowershellWmiCpuInfoParser.cs new file mode 100644 index 0000000000..15ad4d6c77 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/PowershellWmiCpuInfoParser.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; +using Perfolizer.Horology; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +internal static class PowershellWmiCpuInfoParser +{ + internal static CpuInfo Parse(string powershellWmiOutput) + { + HashSet processorModelNames = new HashSet(); + + int physicalCoreCount = 0; + int logicalCoreCount = 0; + int processorCount = 0; + double maxFrequency = 0.0; + double nominalFrequency = 0.0; + + List> processors = SectionsHelper.ParseSectionsForPowershellWmi(powershellWmiOutput, ':'); + foreach (Dictionary processor in processors) + { + if (processor.TryGetValue(WmicCpuInfoKeyNames.NumberOfCores, out string numberOfCoresValue) && + int.TryParse(numberOfCoresValue, out int numberOfCores) && + numberOfCores > 0) + physicalCoreCount += numberOfCores; + + if (processor.TryGetValue(WmicCpuInfoKeyNames.NumberOfLogicalProcessors, out string numberOfLogicalValue) && + int.TryParse(numberOfLogicalValue, out int numberOfLogical) && + numberOfLogical > 0) + logicalCoreCount += numberOfLogical; + + if (processor.TryGetValue(WmicCpuInfoKeyNames.Name, out string name)) + { + processorModelNames.Add(name); + processorCount++; + } + + if (processor.TryGetValue(WmicCpuInfoKeyNames.MaxClockSpeed, out string frequencyValue) + && double.TryParse(frequencyValue, out double frequency) + && frequency > 0) + { + nominalFrequency = nominalFrequency == 0 ? frequency : Math.Min(nominalFrequency, frequency); + maxFrequency = Math.Max(maxFrequency, frequency); + } + } + + string? processorName = processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null; + Frequency? maxFrequencyActual = maxFrequency > 0 && processorCount > 0 + ? Frequency.FromMHz(maxFrequency) : null; + + Frequency? nominalFrequencyActual = nominalFrequency > 0 && processorCount > 0 ? + Frequency.FromMHz(nominalFrequency) : null; + + return new CpuInfo + { + ProcessorName = processorName, + PhysicalProcessorCount = processorCount > 0 ? processorCount : null, + PhysicalCoreCount = physicalCoreCount > 0 ? physicalCoreCount : null, + LogicalCoreCount = logicalCoreCount > 0 ? logicalCoreCount : null, + NominalFrequencyHz = nominalFrequencyActual?.Hertz.RoundToLong(), + MaxFrequencyHz = maxFrequencyActual?.Hertz.RoundToLong() + }; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/WindowsCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WindowsCpuDetector.cs new file mode 100644 index 0000000000..1de238b93f --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WindowsCpuDetector.cs @@ -0,0 +1,4 @@ +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +internal class WindowsCpuDetector() : CpuDetector(new MosCpuDetector(), new PowershellWmiCpuDetector(), + new WmicCpuDetector()); \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuDetector.cs new file mode 100644 index 0000000000..fe3434ae7f --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuDetector.cs @@ -0,0 +1,36 @@ +using System.IO; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Portability; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +/// +/// CPU information from output of the `wmic cpu get Name, NumberOfCores, NumberOfLogicalProcessors /Format:List` command. +/// Windows only. +/// +/// WMIC is deprecated by Microsoft starting with Windows 10 21H1 (including Windows Server), and it is not known whether it still ships with Windows by default. +/// WMIC may be removed in a future version of Windows. See +internal class WmicCpuDetector : ICpuDetector +{ + private const string DefaultWmicPath = @"C:\Windows\System32\wbem\WMIC.exe"; + + public bool IsApplicable() => OsDetector.IsWindows(); + + public CpuInfo? Detect() + { + if (!IsApplicable()) return null; + + const string argList = $"{WmicCpuInfoKeyNames.Name}, " + + $"{WmicCpuInfoKeyNames.NumberOfCores}, " + + $"{WmicCpuInfoKeyNames.NumberOfLogicalProcessors}, " + + $"{WmicCpuInfoKeyNames.MaxClockSpeed}"; + string wmicPath = File.Exists(DefaultWmicPath) ? DefaultWmicPath : "wmic"; + string? wmicOutput = ProcessHelper.RunAndReadOutput(wmicPath, $"cpu get {argList} /Format:List"); + + if (string.IsNullOrEmpty(wmicOutput)) + return null; + + return WmicCpuInfoParser.Parse(wmicOutput); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoKeyNames.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoKeyNames.cs new file mode 100644 index 0000000000..65d8ecf7d8 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoKeyNames.cs @@ -0,0 +1,9 @@ +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +internal static class WmicCpuInfoKeyNames +{ + internal const string NumberOfLogicalProcessors = "NumberOfLogicalProcessors"; + internal const string NumberOfCores = "NumberOfCores"; + internal const string Name = "Name"; + internal const string MaxClockSpeed = "MaxClockSpeed"; +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoParser.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoParser.cs new file mode 100644 index 0000000000..295d76262e --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoParser.cs @@ -0,0 +1,70 @@ +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; +using Perfolizer.Horology; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +internal static class WmicCpuInfoParser +{ + /// + /// Parses wmic output and returns + /// + /// Output of `wmic cpu get Name, NumberOfCores, NumberOfLogicalProcessors /Format:List` + internal static CpuInfo Parse(string wmicOutput) + { + HashSet processorModelNames = new HashSet(); + int physicalCoreCount = 0; + int logicalCoreCount = 0; + int processorsCount = 0; + double maxFrequency = 0.0; + double nominalFrequency = 0.0; + + List> processors = SectionsHelper.ParseSections(wmicOutput, '='); + foreach (var processor in processors) + { + if (processor.TryGetValue(WmicCpuInfoKeyNames.NumberOfCores, out string numberOfCoresValue) && + int.TryParse(numberOfCoresValue, out int numberOfCores) && + numberOfCores > 0) + physicalCoreCount += numberOfCores; + + if (processor.TryGetValue(WmicCpuInfoKeyNames.NumberOfLogicalProcessors, out string numberOfLogicalValue) && + int.TryParse(numberOfLogicalValue, out int numberOfLogical) && + numberOfLogical > 0) + logicalCoreCount += numberOfLogical; + + if (processor.TryGetValue(WmicCpuInfoKeyNames.Name, out string name)) + { + processorModelNames.Add(name); + processorsCount++; + } + + if (processor.TryGetValue(WmicCpuInfoKeyNames.MaxClockSpeed, out string frequencyValue) + && double.TryParse(frequencyValue, out double frequency) + && frequency > 0) + { + nominalFrequency = nominalFrequency == 0 ? frequency : Math.Min(nominalFrequency, frequency); + maxFrequency = Math.Max(maxFrequency, frequency); + } + } + + string? processorName = processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null; + Frequency? maxFrequencyActual = maxFrequency > 0 && processorsCount > 0 + ? Frequency.FromMHz(maxFrequency) + : null; + + Frequency? nominalFrequencyActual = nominalFrequency > 0 && processorsCount > 0 ? Frequency.FromMHz(nominalFrequency) : null; + + return new CpuInfo + { + ProcessorName = processorName, + PhysicalProcessorCount = processorsCount > 0 ? processorsCount : null, + PhysicalCoreCount = physicalCoreCount > 0 ? physicalCoreCount : null, + LogicalCoreCount = logicalCoreCount > 0 ? logicalCoreCount : null, + NominalFrequencyHz = nominalFrequencyActual?.Hertz.RoundToLong(), + MaxFrequencyHz = maxFrequencyActual?.Hertz.RoundToLong() + }; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/macOS/MacOsCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/macOS/MacOsCpuDetector.cs new file mode 100644 index 0000000000..2f540e82d3 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/macOS/MacOsCpuDetector.cs @@ -0,0 +1,25 @@ +using BenchmarkDotNet.Helpers; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.macOS; + +/// +/// CPU information from output of the `sysctl -a` command. +/// MacOSX only. +/// +internal class MacOsCpuDetector : ICpuDetector +{ + public bool IsApplicable() => OsDetector.IsMacOS(); + + public CpuInfo? Detect() + { + if (!IsApplicable()) return null; + + string? sysctlOutput = ProcessHelper.RunAndReadOutput("sysctl", "-a"); + + if (sysctlOutput is null) + return null; + + return SysctlCpuInfoParser.Parse(sysctlOutput); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/macOS/SysctlCpuInfoParser.cs b/src/BenchmarkDotNet/Detectors/Cpu/macOS/SysctlCpuInfoParser.cs new file mode 100644 index 0000000000..5cd4d11c8a --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/macOS/SysctlCpuInfoParser.cs @@ -0,0 +1,60 @@ +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.macOS; + +internal static class SysctlCpuInfoParser +{ + private static class Sysctl + { + internal const string ProcessorName = "machdep.cpu.brand_string"; + internal const string PhysicalProcessorCount = "hw.packages"; + internal const string PhysicalCoreCount = "hw.physicalcpu"; + internal const string LogicalCoreCount = "hw.logicalcpu"; + internal const string NominalFrequency = "hw.cpufrequency"; + internal const string MaxFrequency = "hw.cpufrequency_max"; + } + + /// Output of `sysctl -a` + [SuppressMessage("ReSharper", "StringLiteralTypo")] + internal static CpuInfo Parse(string sysctlOutput) + { + var sysctl = SectionsHelper.ParseSection(sysctlOutput, ':'); + string processorName = sysctl.GetValueOrDefault(Sysctl.ProcessorName); + int? physicalProcessorCount = PositiveIntValue(sysctl, Sysctl.PhysicalProcessorCount); + int? physicalCoreCount = PositiveIntValue(sysctl, Sysctl.PhysicalCoreCount); + int? logicalCoreCount = PositiveIntValue(sysctl, Sysctl.LogicalCoreCount); + long? nominalFrequency = PositiveLongValue(sysctl, Sysctl.NominalFrequency); + long? maxFrequency = PositiveLongValue(sysctl, Sysctl.MaxFrequency); + return new CpuInfo + { + ProcessorName = processorName, + PhysicalProcessorCount = physicalProcessorCount, + PhysicalCoreCount = physicalCoreCount, + LogicalCoreCount = logicalCoreCount, + NominalFrequencyHz = nominalFrequency, + MaxFrequencyHz = maxFrequency + }; + } + + private static int? PositiveIntValue(Dictionary sysctl, string keyName) + { + if (sysctl.TryGetValue(keyName, out string value) && + int.TryParse(value, out int result) && + result > 0) + return result; + return null; + } + + private static long? PositiveLongValue(Dictionary sysctl, string keyName) + { + if (sysctl.TryGetValue(keyName, out string value) && + long.TryParse(value, out long result) && + result > 0) + return result; + return null; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/CpuDetector.cs b/src/BenchmarkDotNet/Detectors/CpuDetector.cs new file mode 100644 index 0000000000..0cb583a95a --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/CpuDetector.cs @@ -0,0 +1,29 @@ +using System; +using System.Linq; +using BenchmarkDotNet.Detectors.Cpu; +using BenchmarkDotNet.Detectors.Cpu.Linux; +using BenchmarkDotNet.Detectors.Cpu.macOS; +using BenchmarkDotNet.Detectors.Cpu.Windows; +using BenchmarkDotNet.Extensions; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors; + +public class CpuDetector(params ICpuDetector[] detectors) : ICpuDetector +{ + public static CpuDetector CrossPlatform => new( + new WindowsCpuDetector(), + new LinuxCpuDetector(), + new MacOsCpuDetector()); + + private static readonly Lazy LazyCpu = new(() => CrossPlatform.Detect()); + public static CpuInfo? Cpu => LazyCpu.Value; + + public bool IsApplicable() => detectors.Any(loader => loader.IsApplicable()); + + public CpuInfo? Detect() => detectors + .Where(loader => loader.IsApplicable()) + .Select(loader => loader.Detect()) + .WhereNotNull() + .FirstOrDefault(); +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/OsDetector.cs b/src/BenchmarkDotNet/Detectors/OsDetector.cs new file mode 100644 index 0000000000..5deeea03a6 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/OsDetector.cs @@ -0,0 +1,178 @@ +using System; +using System.Collections.Generic; +using System.IO; +using BenchmarkDotNet.Helpers; +using Microsoft.Win32; +using System.Runtime.InteropServices; +using BenchmarkDotNet.Extensions; +using Perfolizer.Models; +using static System.Runtime.InteropServices.RuntimeInformation; +using RuntimeEnvironment = Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment; + +namespace BenchmarkDotNet.Detectors; + +public class OsDetector +{ + public static readonly OsDetector Instance = new(); + private OsDetector() { } + + internal static string ExecutableExtension => IsWindows() ? ".exe" : string.Empty; + internal static string ScriptFileExtension => IsWindows() ? ".bat" : ".sh"; + + private readonly Lazy os = new(ResolveOs); + public static OsInfo GetOs() => Instance.os.Value; + + private static OsInfo ResolveOs() + { + if (IsMacOS()) + { + string systemVersion = ExternalToolsHelper.MacSystemProfilerData.Value.GetValueOrDefault("System Version") ?? ""; + string kernelVersion = ExternalToolsHelper.MacSystemProfilerData.Value.GetValueOrDefault("Kernel Version") ?? ""; + return new OsInfo + { + Name = "macOS", + Version = systemVersion, + KernelVersion = kernelVersion + }; + } + + if (IsLinux()) + { + try + { + string version = LinuxOsReleaseHelper.GetNameByOsRelease(File.ReadAllLines("/etc/os-release")); + bool wsl = IsUnderWsl(); + return new OsInfo + { + Name = "Linux", + Version = version, + Container = wsl ? "WSL" : null + }; + } + catch (Exception) + { + // Ignore + } + } + + string operatingSystem = RuntimeEnvironment.OperatingSystem; + string operatingSystemVersion = RuntimeEnvironment.OperatingSystemVersion; + if (IsWindows()) + { + int? ubr = GetWindowsUbr(); + if (ubr != null) + operatingSystemVersion += $".{ubr}"; + } + return new OsInfo + { + Name = operatingSystem, + Version = operatingSystemVersion + }; + } + + private static bool IsUnderWsl() + { + if (!IsLinux()) + return false; + try + { + return File.Exists("/proc/sys/fs/binfmt_misc/WSLInterop"); // https://superuser.com/a/1749811 + } + catch (Exception) + { + return false; + } + } + + // TODO: Introduce a common util API for registry calls, use it also in BenchmarkDotNet.Toolchains.CsProj.GetCurrentVersionBasedOnWindowsRegistry + /// + /// On Windows, this method returns UBR (Update Build Revision) based on Registry. + /// Returns null if the value is not available + /// + /// + private static int? GetWindowsUbr() + { + if (IsWindows()) + { + try + { + using (var baseKey = RegistryKey.OpenBaseKey(RegistryHive.LocalMachine, RegistryView.Registry32)) + using (var ndpKey = baseKey.OpenSubKey(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion")) + { + if (ndpKey == null) + return null; + + return Convert.ToInt32(ndpKey.GetValue("UBR")); + } + } + catch (Exception) + { + return null; + } + } + return null; + } + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("windows")] +#endif + internal static bool IsWindows() => +#if NET6_0_OR_GREATER + OperatingSystem.IsWindows(); // prefer linker-friendly OperatingSystem APIs +#else + IsOSPlatform(OSPlatform.Windows); +#endif + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("linux")] +#endif + internal static bool IsLinux() => +#if NET6_0_OR_GREATER + OperatingSystem.IsLinux(); +#else + IsOSPlatform(OSPlatform.Linux); +#endif + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("macos")] +#endif + // ReSharper disable once InconsistentNaming + internal static bool IsMacOS() => +#if NET6_0_OR_GREATER + OperatingSystem.IsMacOS(); +#else + IsOSPlatform(OSPlatform.OSX); +#endif + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("android")] +#endif + internal static bool IsAndroid() => +#if NET6_0_OR_GREATER + OperatingSystem.IsAndroid(); +#else + Type.GetType("Java.Lang.Object, Mono.Android") != null; +#endif + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("ios")] +#endif + // ReSharper disable once InconsistentNaming + internal static bool IsIOS() => +#if NET6_0_OR_GREATER + OperatingSystem.IsIOS(); +#else + Type.GetType("Foundation.NSObject, Xamarin.iOS") != null; +#endif + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("tvos")] +#endif + // ReSharper disable once InconsistentNaming + internal static bool IsTvOS() => +#if NET6_0_OR_GREATER + OperatingSystem.IsTvOS(); +#else + IsOSPlatform(OSPlatform.Create("TVOS")); +#endif +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/AllocatedMemoryMetricDescriptor.cs b/src/BenchmarkDotNet/Diagnosers/AllocatedMemoryMetricDescriptor.cs index dc6cac925d..634e795680 100644 --- a/src/BenchmarkDotNet/Diagnosers/AllocatedMemoryMetricDescriptor.cs +++ b/src/BenchmarkDotNet/Diagnosers/AllocatedMemoryMetricDescriptor.cs @@ -1,6 +1,7 @@ using System; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Reports; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Diagnosers { @@ -13,7 +14,7 @@ internal class AllocatedMemoryMetricDescriptor : IMetricDescriptor public string Legend => "Allocated memory per single operation (managed only, inclusive, 1KB = 1024B)"; public string NumberFormat => "0.##"; public UnitType UnitType => UnitType.Size; - public string Unit => SizeUnit.B.Name; + public string Unit => SizeUnit.B.Abbreviation; public bool TheGreaterTheBetter => false; public int PriorityInCategory => GC.MaxGeneration + 1; public bool GetIsAvailable(Metric metric) => true; diff --git a/src/BenchmarkDotNet/Diagnosers/AllocatedNativeMemoryDescriptor.cs b/src/BenchmarkDotNet/Diagnosers/AllocatedNativeMemoryDescriptor.cs index 8781333f80..c4cb964355 100644 --- a/src/BenchmarkDotNet/Diagnosers/AllocatedNativeMemoryDescriptor.cs +++ b/src/BenchmarkDotNet/Diagnosers/AllocatedNativeMemoryDescriptor.cs @@ -1,5 +1,6 @@ using BenchmarkDotNet.Columns; using BenchmarkDotNet.Reports; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Diagnosers { @@ -12,7 +13,7 @@ internal class AllocatedNativeMemoryDescriptor : IMetricDescriptor public string Legend => $"Allocated native memory per single operation"; public string NumberFormat => "N0"; public UnitType UnitType => UnitType.Size; - public string Unit => SizeUnit.B.Name; + public string Unit => SizeUnit.B.Abbreviation; public bool TheGreaterTheBetter => false; public int PriorityInCategory => 0; public bool GetIsAvailable(Metric metric) => true; @@ -27,7 +28,7 @@ internal class NativeMemoryLeakDescriptor : IMetricDescriptor public string Legend => $"Native memory leak size in byte."; public string NumberFormat => "N0"; public UnitType UnitType => UnitType.Size; - public string Unit => SizeUnit.B.Name; + public string Unit => SizeUnit.B.Abbreviation; public bool TheGreaterTheBetter => false; public int PriorityInCategory => 0; public bool GetIsAvailable(Metric metric) => true; diff --git a/src/BenchmarkDotNet/Diagnosers/DiagnoserActionParameters.cs b/src/BenchmarkDotNet/Diagnosers/DiagnoserActionParameters.cs index a403e170f5..c7d957b385 100644 --- a/src/BenchmarkDotNet/Diagnosers/DiagnoserActionParameters.cs +++ b/src/BenchmarkDotNet/Diagnosers/DiagnoserActionParameters.cs @@ -6,7 +6,7 @@ namespace BenchmarkDotNet.Diagnosers { public class DiagnoserActionParameters { - public DiagnoserActionParameters(Process process, BenchmarkCase benchmarkCase, BenchmarkId benchmarkId) + public DiagnoserActionParameters(Process? process, BenchmarkCase benchmarkCase, BenchmarkId benchmarkId) { Process = process; BenchmarkCase = benchmarkCase; diff --git a/src/BenchmarkDotNet/Diagnosers/DiagnoserResults.cs b/src/BenchmarkDotNet/Diagnosers/DiagnoserResults.cs index 68591ad8c8..6d7fb7b1db 100644 --- a/src/BenchmarkDotNet/Diagnosers/DiagnoserResults.cs +++ b/src/BenchmarkDotNet/Diagnosers/DiagnoserResults.cs @@ -2,6 +2,7 @@ using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.Results; +using System.Collections.Generic; using System.Linq; namespace BenchmarkDotNet.Diagnosers @@ -16,6 +17,7 @@ public DiagnoserResults(BenchmarkCase benchmarkCase, ExecuteResult executeResult ThreadingStats = executeResult.ThreadingStats; BuildResult = buildResult; ExceptionFrequency = executeResult.ExceptionFrequency; + Measurements = executeResult.Measurements; } public BenchmarkCase BenchmarkCase { get; } @@ -29,5 +31,7 @@ public DiagnoserResults(BenchmarkCase benchmarkCase, ExecuteResult executeResult public double ExceptionFrequency { get; } public BuildResult BuildResult { get; } + + public IReadOnlyList Measurements { get; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs b/src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs index 1410aad8bc..40e8f231e7 100644 --- a/src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs +++ b/src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs @@ -5,6 +5,7 @@ using System.Reflection; using System.Threading; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; @@ -40,11 +41,11 @@ private static IEnumerable LoadDiagnosers() { yield return EventPipeProfiler.Default; - if (RuntimeInformation.IsLinux()) + if (OsDetector.IsLinux()) yield return PerfCollectProfiler.Default; } - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) yield break; foreach (var windowsDiagnoser in LoadWindowsDiagnosers()) @@ -80,6 +81,11 @@ private static IDiagnoser[] LoadWindowsDiagnosers() CreateDiagnoser(diagnosticsAssembly, "BenchmarkDotNet.Diagnostics.Windows.NativeMemoryProfiler") }; } + catch (Exception ex) when (ex is FileNotFoundException || ex is BadImageFormatException) + { + // Return an array of UnresolvedDiagnoser objects when the assembly does not contain the requested diagnoser + return new[] { GetUnresolvedDiagnoser() }; + } catch (Exception ex) // we're loading a plug-in, better to be safe rather than sorry { ConsoleLogger.Default.WriteLineError($"Error loading {WindowsDiagnosticAssemblyFileName}: {ex.GetType().Name} - {ex.Message}"); diff --git a/src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs index 782e895d3e..574d1e54f1 100644 --- a/src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs @@ -1,4 +1,5 @@ using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; @@ -14,9 +15,11 @@ namespace BenchmarkDotNet.Diagnosers { public class ExceptionDiagnoser : IDiagnoser { - public static readonly ExceptionDiagnoser Default = new ExceptionDiagnoser(); + public static readonly ExceptionDiagnoser Default = new ExceptionDiagnoser(new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: true)); - private ExceptionDiagnoser() { } + public ExceptionDiagnoser(ExceptionDiagnoserConfig config) => Config = config; + + public ExceptionDiagnoserConfig Config { get; } public IEnumerable Ids => new[] { nameof(ExceptionDiagnoser) }; @@ -32,14 +35,18 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { } public IEnumerable ProcessResults(DiagnoserResults results) { - yield return new Metric(ExceptionsFrequencyMetricDescriptor.Instance, results.ExceptionFrequency); + yield return new Metric(new ExceptionsFrequencyMetricDescriptor(Config), results.ExceptionFrequency); } public IEnumerable Validate(ValidationParameters validationParameters) => Enumerable.Empty(); - private class ExceptionsFrequencyMetricDescriptor : IMetricDescriptor + internal class ExceptionsFrequencyMetricDescriptor : IMetricDescriptor { - internal static readonly IMetricDescriptor Instance = new ExceptionsFrequencyMetricDescriptor(); + public ExceptionDiagnoserConfig Config { get; } + public ExceptionsFrequencyMetricDescriptor(ExceptionDiagnoserConfig config = null) + { + Config = config; + } public string Id => "ExceptionFrequency"; public string DisplayName => Column.Exceptions; @@ -49,7 +56,13 @@ private class ExceptionsFrequencyMetricDescriptor : IMetricDescriptor public string Unit => "Count"; public bool TheGreaterTheBetter => false; public int PriorityInCategory => 0; - public bool GetIsAvailable(Metric metric) => true; + public bool GetIsAvailable(Metric metric) + { + if (Config == null) + return metric.Value > 0; + else + return Config.DisplayExceptionsIfZeroValue || metric.Value > 0; + } } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs b/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs index 6dd5bdd2e9..9a3d348665 100644 --- a/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs +++ b/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Runtime.InteropServices; using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; @@ -31,8 +32,8 @@ public class PerfCollectProfiler : IProfiler private readonly PerfCollectProfilerConfig config; private readonly DateTime creationTime = DateTime.Now; - private readonly Dictionary benchmarkToTraceFile = new (); - private readonly HashSet cliPathWithSymbolsInstalled = new (); + private readonly Dictionary benchmarkToTraceFile = new(); + private readonly HashSet cliPathWithSymbolsInstalled = new(); private FileInfo perfCollectFile; private Process perfCollectProcess; @@ -53,7 +54,7 @@ public class PerfCollectProfiler : IProfiler public IEnumerable Validate(ValidationParameters validationParameters) { - if (!RuntimeInformation.IsLinux()) + if (!OsDetector.IsLinux()) { yield return new ValidationError(true, "The PerfCollectProfiler works only on Linux!"); yield break; @@ -227,7 +228,7 @@ private void EnsureSymbolsForNativeRuntime(DiagnoserActionParameters parameters) ILogger logger = parameters.Config.GetCompositeLogger(); // We install the tool in a dedicated directory in order to always use latest version and avoid issues with broken existing configs. string toolPath = Path.Combine(Path.GetTempPath(), "BenchmarkDotNet", "symbols"); - DotNetCliCommand cliCommand = new ( + DotNetCliCommand cliCommand = new( cliPath: cliPath, arguments: $"tool install dotnet-symbol --tool-path \"{toolPath}\"", generateResult: null, @@ -252,7 +253,7 @@ private void EnsureSymbolsForNativeRuntime(DiagnoserActionParameters parameters) } private FileInfo GetTraceFile(DiagnoserActionParameters parameters, string extension) - => new (ArtifactFileNameHelper.GetTraceFilePath(parameters, creationTime, extension) + => new(ArtifactFileNameHelper.GetTraceFilePath(parameters, creationTime, extension) .Replace(" ", "_")); // perfcollect does not allow for spaces in the trace file name } } diff --git a/src/BenchmarkDotNet/Diagnosers/SnapshotProfilerBase.cs b/src/BenchmarkDotNet/Diagnosers/SnapshotProfilerBase.cs new file mode 100644 index 0000000000..3105ceeeb9 --- /dev/null +++ b/src/BenchmarkDotNet/Diagnosers/SnapshotProfilerBase.cs @@ -0,0 +1,195 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.IO; +using System.Linq; +using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Validators; + +namespace BenchmarkDotNet.Diagnosers; + +public abstract class SnapshotProfilerBase : IProfiler +{ + public abstract string ShortName { get; } + + protected abstract void InitTool(Progress progress); + protected abstract void AttachToCurrentProcess(string snapshotFile); + protected abstract void AttachToProcessByPid(int pid, string snapshotFile); + protected abstract void TakeSnapshot(); + protected abstract void Detach(); + + protected abstract string CreateSnapshotFilePath(DiagnoserActionParameters parameters); + protected abstract string GetRunnerPath(); + internal abstract bool IsSupported(RuntimeMoniker runtimeMoniker); + + private readonly List snapshotFilePaths = []; + + public IEnumerable Ids => [ShortName]; + public IEnumerable Exporters => []; + public IEnumerable Analysers => []; + + public RunMode GetRunMode(BenchmarkCase benchmarkCase) => + IsSupported(benchmarkCase.Job.Environment.GetRuntime().RuntimeMoniker) ? RunMode.ExtraRun : RunMode.None; + + public void Handle(HostSignal signal, DiagnoserActionParameters parameters) + { + var logger = parameters.Config.GetCompositeLogger(); + var job = parameters.BenchmarkCase.Job; + + var runtimeMoniker = job.Environment.GetRuntime().RuntimeMoniker; + if (!IsSupported(runtimeMoniker)) + { + logger.WriteLineError($"Runtime '{runtimeMoniker}' is not supported by dotMemory"); + return; + } + + switch (signal) + { + case HostSignal.BeforeAnythingElse: + Init(logger); + break; + case HostSignal.BeforeActualRun: + string snapshotFilePath = Start(logger, parameters); + snapshotFilePaths.Add(snapshotFilePath); + break; + case HostSignal.AfterActualRun: + Stop(logger); + break; + } + } + + public IEnumerable Validate(ValidationParameters validationParameters) + { + var runtimeMonikers = validationParameters.Benchmarks.Select(b => b.Job.Environment.GetRuntime().RuntimeMoniker).Distinct(); + foreach (var runtimeMoniker in runtimeMonikers) + if (!IsSupported(runtimeMoniker)) + yield return new ValidationError(true, $"Runtime '{runtimeMoniker}' is not supported by dotMemory"); + } + + public IEnumerable ProcessResults(DiagnoserResults results) => ImmutableArray.Empty; + + public void DisplayResults(ILogger logger) + { + if (snapshotFilePaths.Count != 0) + { + logger.WriteLineInfo($"The following {ShortName} snapshots were generated:"); + foreach (string snapshotFilePath in snapshotFilePaths) + logger.WriteLineInfo($"* {snapshotFilePath}"); + } + } + + private void Init(ILogger logger) + { + try + { + logger.WriteLineInfo($"Ensuring that {ShortName} prerequisite is installed..."); + var progress = new Progress(logger, $"Installing {ShortName}"); + InitTool(progress); + logger.WriteLineInfo($"{ShortName} prerequisite is installed"); + logger.WriteLineInfo($"{ShortName} runner path: {GetRunnerPath()}"); + } + catch (Exception e) + { + logger.WriteLineError(e.ToString()); + } + } + + private string Start(ILogger logger, DiagnoserActionParameters parameters) + { + string snapshotFilePath = CreateSnapshotFilePath(parameters); + string? snapshotDirectory = Path.GetDirectoryName(snapshotFilePath); + logger.WriteLineInfo($"Target snapshot file: {snapshotFilePath}"); + if (!Directory.Exists(snapshotDirectory) && snapshotDirectory != null) + { + try + { + Directory.CreateDirectory(snapshotDirectory); + } + catch (Exception e) + { + logger.WriteLineError($"Failed to create directory: {snapshotDirectory}"); + logger.WriteLineError(e.ToString()); + } + } + + try + { + logger.WriteLineInfo($"Attaching {ShortName} to the process..."); + Attach(parameters, snapshotFilePath); + logger.WriteLineInfo($"{ShortName} is successfully attached"); + } + catch (Exception e) + { + logger.WriteLineError(e.ToString()); + return snapshotFilePath; + } + + return snapshotFilePath; + } + + private void Stop(ILogger logger) + { + try + { + logger.WriteLineInfo($"Taking {ShortName} snapshot..."); + TakeSnapshot(); + logger.WriteLineInfo($"{ShortName} snapshot is successfully taken"); + } + catch (Exception e) + { + logger.WriteLineError(e.ToString()); + } + + try + { + logger.WriteLineInfo($"Detaching {ShortName} from the process..."); + Detach(); + logger.WriteLineInfo($"{ShortName} is successfully detached"); + } + catch (Exception e) + { + logger.WriteLineError(e.ToString()); + } + } + + + private void Attach(DiagnoserActionParameters parameters, string snapshotFile) + { + int pid = parameters.Process.Id; + int currentPid = Process.GetCurrentProcess().Id; + if (pid != currentPid) + AttachToProcessByPid(pid, snapshotFile); + else + AttachToCurrentProcess(snapshotFile); + } + + protected class Progress(ILogger logger, string title) : IProgress + { + private static readonly TimeSpan ReportInterval = TimeSpan.FromSeconds(0.1); + + private int lastProgress; + private Stopwatch? stopwatch; + + public void Report(double value) + { + int progress = (int)Math.Floor(value); + bool needToReport = stopwatch == null || + (stopwatch != null && stopwatch?.Elapsed > ReportInterval) || + progress == 100; + + if (lastProgress != progress && needToReport) + { + logger.WriteLineInfo($"{title}: {progress}%"); + lastProgress = progress; + stopwatch = Stopwatch.StartNew(); + } + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs index 057a0bb624..36b2f942c3 100644 --- a/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs @@ -15,9 +15,10 @@ namespace BenchmarkDotNet.Diagnosers { public class ThreadingDiagnoser : IDiagnoser { - public static readonly ThreadingDiagnoser Default = new ThreadingDiagnoser(); + public static readonly ThreadingDiagnoser Default = new ThreadingDiagnoser(new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: true, displayLockContentionWhenZero: true)); - private ThreadingDiagnoser() { } + public ThreadingDiagnoser(ThreadingDiagnoserConfig config) => Config = config; + public ThreadingDiagnoserConfig Config { get; } public IEnumerable Ids => new[] { nameof(ThreadingDiagnoser) }; @@ -33,8 +34,9 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { } public IEnumerable ProcessResults(DiagnoserResults results) { - yield return new Metric(CompletedWorkItemCountMetricDescriptor.Instance, results.ThreadingStats.CompletedWorkItemCount / (double)results.ThreadingStats.TotalOperations); - yield return new Metric(LockContentionCountMetricDescriptor.Instance, results.ThreadingStats.LockContentionCount / (double)results.ThreadingStats.TotalOperations); + + yield return new Metric(new CompletedWorkItemCountMetricDescriptor(Config), results.ThreadingStats.CompletedWorkItemCount / (double)results.ThreadingStats.TotalOperations); + yield return new Metric(new LockContentionCountMetricDescriptor(Config), results.ThreadingStats.LockContentionCount / (double)results.ThreadingStats.TotalOperations); } public IEnumerable Validate(ValidationParameters validationParameters) @@ -50,10 +52,15 @@ public IEnumerable Validate(ValidationParameters validationPara } } - private class CompletedWorkItemCountMetricDescriptor : IMetricDescriptor + internal class CompletedWorkItemCountMetricDescriptor : IMetricDescriptor { internal static readonly IMetricDescriptor Instance = new CompletedWorkItemCountMetricDescriptor(); + private ThreadingDiagnoserConfig Config { get; } + public CompletedWorkItemCountMetricDescriptor(ThreadingDiagnoserConfig config = null) + { + Config = config; + } public string Id => "CompletedWorkItemCount"; public string DisplayName => Column.CompletedWorkItems; public string Legend => "The number of work items that have been processed in ThreadPool (per single operation)"; @@ -62,13 +69,26 @@ private class CompletedWorkItemCountMetricDescriptor : IMetricDescriptor public string Unit => "Count"; public bool TheGreaterTheBetter => false; public int PriorityInCategory => 0; - public bool GetIsAvailable(Metric metric) => true; + public bool GetIsAvailable(Metric metric) + { + if (Config == null) + return metric.Value > 0; + else + return Config.DisplayCompletedWorkItemCountWhenZero || metric.Value > 0; + } } - private class LockContentionCountMetricDescriptor : IMetricDescriptor + internal class LockContentionCountMetricDescriptor : IMetricDescriptor { internal static readonly IMetricDescriptor Instance = new LockContentionCountMetricDescriptor(); + private ThreadingDiagnoserConfig Config { get; } + + public LockContentionCountMetricDescriptor(ThreadingDiagnoserConfig config = null) + { + Config = config; + } + public string Id => "LockContentionCount"; public string DisplayName => Column.LockContentions; public string Legend => "The number of times there was contention upon trying to take a Monitor's lock (per single operation)"; @@ -77,7 +97,13 @@ private class LockContentionCountMetricDescriptor : IMetricDescriptor public string Unit => "Count"; public bool TheGreaterTheBetter => false; public int PriorityInCategory => 0; - public bool GetIsAvailable(Metric metric) => true; + public bool GetIsAvailable(Metric metric) + { + if (Config == null) + return metric.Value > 0; + else + return Config.DisplayLockContentionWhenZero || metric.Value > 0; + } } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoserConfig.cs b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoserConfig.cs new file mode 100644 index 0000000000..6af3da25ac --- /dev/null +++ b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoserConfig.cs @@ -0,0 +1,23 @@ +using JetBrains.Annotations; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BenchmarkDotNet.Diagnosers +{ + public class ThreadingDiagnoserConfig + { + /// Display configuration for 'LockContentionCount' when it is empty. True (displayed) by default. + /// Display configuration for 'CompletedWorkItemCount' when it is empty. True (displayed) by default. + + [PublicAPI] + public ThreadingDiagnoserConfig(bool displayLockContentionWhenZero = true, bool displayCompletedWorkItemCountWhenZero = true) + { + DisplayLockContentionWhenZero = displayLockContentionWhenZero; + DisplayCompletedWorkItemCountWhenZero = displayCompletedWorkItemCountWhenZero; + } + + public bool DisplayLockContentionWhenZero { get; } + public bool DisplayCompletedWorkItemCountWhenZero { get; } + } +} diff --git a/src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs index f00bb4ed67..43cc910559 100644 --- a/src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Loggers; @@ -31,10 +32,10 @@ public IEnumerable Validate(ValidationParameters validationPara => new[] { new ValidationError(false, GetErrorMessage()) }; private string GetErrorMessage() => $@"Unable to resolve {unresolved.Name} diagnoser using dynamic assembly loading. - {(RuntimeInformation.IsFullFramework || RuntimeInformation.IsWindows() + {(RuntimeInformation.IsFullFramework || OsDetector.IsWindows() ? "Please make sure that you have installed the latest BenchmarkDotNet.Diagnostics.Windows package. " + Environment.NewLine + "If you are using `dotnet build` you also need to consume one of its public types to make sure that MSBuild copies it to the output directory. " + "The alternative is to use `true` in your project file." - : $"Please make sure that it's supported on {RuntimeInformation.GetOsVersion()}")}"; + : $"Please make sure that it's supported on {OsDetector.GetOs()}")}"; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs b/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs index 16ebbde219..7c10765b8e 100644 --- a/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs @@ -139,7 +139,7 @@ public void Feed(Arm64Instruction instruction) public Arm64RegisterId RegisterId { get { return _registerId; } } } - internal class Arm64Disassembler : ClrMdV2Disassembler + internal class Arm64Disassembler : ClrMdV3Disassembler { internal sealed class RuntimeSpecificData { @@ -189,7 +189,7 @@ internal RuntimeSpecificData(State state) } } - private static readonly Dictionary runtimeSpecificData = new (); + private static readonly Dictionary runtimeSpecificData = new(); protected override IEnumerable Decode(byte[] code, ulong startAddress, State state, int depth, ClrMethod currentMethod, DisassemblySyntax syntax) { diff --git a/src/BenchmarkDotNet/Disassemblers/Arm64InstructionFormatter.cs b/src/BenchmarkDotNet/Disassemblers/Arm64InstructionFormatter.cs index deb557256c..5426dcfdd9 100644 --- a/src/BenchmarkDotNet/Disassemblers/Arm64InstructionFormatter.cs +++ b/src/BenchmarkDotNet/Disassemblers/Arm64InstructionFormatter.cs @@ -12,7 +12,7 @@ internal static class Arm64InstructionFormatter internal static string Format(Arm64Asm asm, FormatterOptions formatterOptions, bool printInstructionAddresses, uint pointerSize, IReadOnlyDictionary symbols) { - StringBuilder output = new (); + StringBuilder output = new(); Arm64Instruction instruction = asm.Instruction; if (printInstructionAddresses) diff --git a/src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs b/src/BenchmarkDotNet/Disassemblers/ClrMdV3Disassembler.cs similarity index 75% rename from src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs rename to src/BenchmarkDotNet/Disassemblers/ClrMdV3Disassembler.cs index 5a0ddccafb..6bd01426d8 100644 --- a/src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/ClrMdV3Disassembler.cs @@ -7,23 +7,25 @@ using System.IO; using System.Linq; using System.Text.RegularExpressions; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Disassemblers { - // This Disassembler uses ClrMd v2x. Please keep it in sync with ClrMdV1Disassembler (if possible). - internal abstract class ClrMdV2Disassembler + // This Disassembler uses ClrMd v3x. Please keep it in sync with ClrMdV1Disassembler (if possible). + internal abstract class ClrMdV3Disassembler + { private static readonly ulong MinValidAddress = GetMinValidAddress(); private static ulong GetMinValidAddress() { // https://github.com/dotnet/BenchmarkDotNet/pull/2413#issuecomment-1688100117 - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) return ushort.MaxValue + 1; - if (RuntimeInformation.IsLinux()) + if (OsDetector.IsLinux()) return (ulong) Environment.SystemPageSize; - if (RuntimeInformation.IsMacOS()) + if (OsDetector.IsMacOS()) return RuntimeInformation.GetCurrentPlatform() switch { Environments.Platform.X86 or Environments.Platform.X64 => 4096, @@ -64,7 +66,7 @@ internal DisassemblyResult AttachAndDisassemble(Settings settings) state.Todo.Enqueue( new MethodInfo( // the Disassembler Entry Method is always parameterless, so check by name is enough - typeWithBenchmark.Methods.Single(method => method.IsPublic && method.Name == settings.MethodName), + typeWithBenchmark.Methods.Single(method => method.Attributes.HasFlag(System.Reflection.MethodAttributes.Public) && method.Name == settings.MethodName), 0)); } @@ -126,6 +128,7 @@ private DisassembledMethod[] Disassemble(Settings settings, State state) var result = new List(); DisassemblySyntax syntax = (DisassemblySyntax)Enum.Parse(typeof(DisassemblySyntax), settings.Syntax); + using var sourceCodeProvider = new SourceCodeProvider(); while (state.Todo.Count != 0) { var methodInfo = state.Todo.Dequeue(); @@ -134,7 +137,7 @@ private DisassembledMethod[] Disassemble(Settings settings, State state) continue; // already handled if (settings.MaxDepth >= methodInfo.Depth) - result.Add(DisassembleMethod(methodInfo, state, settings, syntax)); + result.Add(DisassembleMethod(methodInfo, state, settings, syntax, sourceCodeProvider)); } return result.ToArray(); @@ -142,15 +145,16 @@ private DisassembledMethod[] Disassemble(Settings settings, State state) private static bool CanBeDisassembled(ClrMethod method) => method.ILOffsetMap.Length > 0 && method.NativeCode > 0; - private DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State state, Settings settings, DisassemblySyntax syntax) + private DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State state, Settings settings, DisassemblySyntax syntax, SourceCodeProvider sourceCodeProvider) { var method = methodInfo.Method; if (!CanBeDisassembled(method)) { - if (method.IsPInvoke) + if (method.Attributes.HasFlag(System.Reflection.MethodAttributes.PinvokeImpl)) return CreateEmpty(method, "PInvoke method"); - if (method.IL is null || method.IL.Length == 0) + var ilInfo = method.GetILInfo(); + if (ilInfo is null || ilInfo.Length == 0) return CreateEmpty(method, "Extern method"); if (method.CompilationType == MethodCompilationType.None) return CreateEmpty(method, "Method was not JITted yet."); @@ -165,7 +169,7 @@ private DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State state, var uniqueSourceCodeLines = new HashSet(new SharpComparer()); // for getting C# code we always use the original ILOffsetMap foreach (var map in method.ILOffsetMap.Where(map => map.StartAddress < map.EndAddress && map.ILOffset >= 0).OrderBy(map => map.StartAddress)) - foreach (var sharp in SourceCodeProvider.GetSource(method, map)) + foreach (var sharp in sourceCodeProvider.GetSource(method, map)) uniqueSourceCodeLines.Add(sharp); codes.AddRange(uniqueSourceCodeLines); @@ -213,60 +217,30 @@ private IEnumerable Decode(ILToNativeMap map, State state, int depth, ClrMe private static ILToNativeMap[] GetCompleteNativeMap(ClrMethod method, ClrRuntime runtime) { - if (!TryReadNativeCodeAddresses(runtime, method, out ulong startAddress, out ulong endAddress)) + // it's better to use one single map rather than few small ones + // it's simply easier to get next instruction when decoding ;) + + var hotColdInfo = method.HotColdInfo; + if (hotColdInfo.HotSize > 0 && hotColdInfo.HotStart > 0) { - startAddress = method.NativeCode; - endAddress = ulong.MaxValue; + return hotColdInfo.ColdSize <= 0 + ? new[] { new ILToNativeMap() { StartAddress = hotColdInfo.HotStart, EndAddress = hotColdInfo.HotStart + hotColdInfo.HotSize, ILOffset = -1 } } + : new[] + { + new ILToNativeMap() { StartAddress = hotColdInfo.HotStart, EndAddress = hotColdInfo.HotStart + hotColdInfo.HotSize, ILOffset = -1 }, + new ILToNativeMap() { StartAddress = hotColdInfo.ColdStart, EndAddress = hotColdInfo.ColdStart + hotColdInfo.ColdSize, ILOffset = -1 } + }; } - ILToNativeMap[] sortedMaps = method.ILOffsetMap // CanBeDisassembled ensures that there is at least one map in ILOffsetMap - .Where(map => map.StartAddress >= startAddress && map.StartAddress < endAddress) // can be false for Tier 0 maps, EndAddress is not checked on purpose here - .Where(map => map.StartAddress < map.EndAddress) // some maps have 0 length (they don't have corresponding assembly code?) + return method.ILOffsetMap + .Where(map => map.StartAddress < map.EndAddress) // some maps have 0 length? .OrderBy(map => map.StartAddress) // we need to print in the machine code order, not IL! #536 - .Select(map => new ILToNativeMap() - { - StartAddress = map.StartAddress, - // some maps have EndAddress > codeHeaderData.MethodStart + codeHeaderData.MethodSize and contain garbage (#2074). They need to be fixed! - EndAddress = Math.Min(map.EndAddress, endAddress), - ILOffset = map.ILOffset - }) .ToArray(); - - if (sortedMaps.Length == 0) - { - // In such situation ILOffsetMap most likely describes Tier 0, while CodeHeaderData Tier 1. - // Since we care about Tier 1 (if it's present), we "fake" a Tier 1 map. - return new[] { new ILToNativeMap() { StartAddress = startAddress, EndAddress = endAddress } }; - } - else if (sortedMaps[0].StartAddress != startAddress || (sortedMaps[sortedMaps.Length - 1].EndAddress != endAddress && endAddress != ulong.MaxValue)) - { - // In such situation ILOffsetMap most likely is missing few bytes. We just "extend" it to avoid producing "bad" instructions. - return new[] { new ILToNativeMap() { StartAddress = startAddress, EndAddress = endAddress } }; - } - - return sortedMaps; } private static DisassembledMethod CreateEmpty(ClrMethod method, string reason) => DisassembledMethod.Empty(method.Signature, method.NativeCode, reason); - protected static bool TryReadNativeCodeAddresses(ClrRuntime runtime, ClrMethod method, out ulong startAddress, out ulong endAddress) - { - if (method is not null - && runtime.DacLibrary.SOSDacInterface.GetCodeHeaderData(method.NativeCode, out var codeHeaderData) == HResult.S_OK - && codeHeaderData.MethodSize > 0) // false for extern methods! - { - // HotSize can be missing or be invalid (https://github.com/microsoft/clrmd/issues/1036). - // So we fetch the method size on our own. - startAddress = codeHeaderData.MethodStart; - endAddress = codeHeaderData.MethodStart + codeHeaderData.MethodSize; - return true; - } - - startAddress = endAddress = 0; - return false; - } - protected void TryTranslateAddressToName(ulong address, bool isAddressPrecodeMD, State state, int depth, ClrMethod currentMethod) { if (!IsValidAddress(address) || state.AddressToNameMapping.ContainsKey(address)) @@ -282,18 +256,10 @@ protected void TryTranslateAddressToName(ulong address, bool isAddressPrecodeMD, } var method = runtime.GetMethodByInstructionPointer(address); - if (method is null && (address & ((uint) runtime.DataTarget.DataReader.PointerSize - 1)) == 0) - { - if (runtime.DataTarget.DataReader.ReadPointer(address, out ulong newAddress) && IsValidAddress(newAddress)) - { - method = runtime.GetMethodByInstructionPointer(newAddress); - - method = WorkaroundGetMethodByInstructionPointerBug(runtime, method, newAddress); - } - } - else + if (method is null && (address & ((uint) runtime.DataTarget.DataReader.PointerSize - 1)) == 0 + && runtime.DataTarget.DataReader.ReadPointer(address, out ulong newAddress) && IsValidAddress(newAddress)) { - method = WorkaroundGetMethodByInstructionPointerBug(runtime, method, address); + method = runtime.GetMethodByInstructionPointer(newAddress); } if (method is null) @@ -312,7 +278,7 @@ protected void TryTranslateAddressToName(ulong address, bool isAddressPrecodeMD, return; } - var methodTableName = runtime.DacLibrary.SOSDacInterface.GetMethodTableName(address); + var methodTableName = runtime.GetTypeByMethodTable(address)?.Name; if (!string.IsNullOrEmpty(methodTableName)) { state.AddressToNameMapping.Add(address, $"MT_{methodTableName}"); @@ -334,7 +300,7 @@ protected void TryTranslateAddressToName(ulong address, bool isAddressPrecodeMD, protected void FlushCachedDataIfNeeded(IDataReader dataTargetDataReader, ulong address, byte[] buffer) { - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) { if (dataTargetDataReader.Read(address, buffer) <= 0) { @@ -348,13 +314,6 @@ protected void FlushCachedDataIfNeeded(IDataReader dataTargetDataReader, ulong a } } - // GetMethodByInstructionPointer sometimes returns wrong methods. - // In case given address does not belong to the methods range, null is returned. - private static ClrMethod WorkaroundGetMethodByInstructionPointerBug(ClrRuntime runtime, ClrMethod method, ulong newAddress) - => TryReadNativeCodeAddresses(runtime, method, out ulong startAddress, out ulong endAddress) && !(startAddress >= newAddress && newAddress <= endAddress) - ? null - : method; - private class SharpComparer : IEqualityComparer { public bool Equals(Sharp x, Sharp y) diff --git a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs index 31e7570e86..7d0f95326c 100644 --- a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs +++ b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs @@ -3,6 +3,7 @@ using System.Linq; using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Disassemblers; using BenchmarkDotNet.Disassemblers.Exporters; using BenchmarkDotNet.Engines; @@ -16,6 +17,7 @@ using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.InProcess.NoEmit; using BenchmarkDotNet.Validators; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Diagnosers { @@ -74,7 +76,7 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) case HostSignal.AfterAll when ShouldUseSameArchitectureDisassembler(benchmark, parameters): results.Add(benchmark, sameArchitectureDisassembler.Disassemble(parameters)); break; - case HostSignal.AfterAll when RuntimeInformation.IsWindows() && !ShouldUseMonoDisassembler(benchmark): + case HostSignal.AfterAll when OsDetector.IsWindows() && !ShouldUseMonoDisassembler(benchmark): results.Add(benchmark, windowsDifferentArchitectureDisassembler.Disassemble(parameters)); break; case HostSignal.SeparateLogic when ShouldUseMonoDisassembler(benchmark): @@ -111,7 +113,7 @@ public IEnumerable Validate(ValidationParameters validationPara if (ShouldUseClrMdDisassembler(benchmark)) { - if (RuntimeInformation.IsLinux()) + if (OsDetector.IsLinux()) { var runtime = benchmark.Job.ResolveValue(EnvironmentMode.RuntimeCharacteristic, EnvironmentResolver.Instance); @@ -142,13 +144,13 @@ private static bool ShouldUseMonoDisassembler(BenchmarkCase benchmarkCase) // when we add macOS support, RuntimeInformation.IsMacOS() needs to be added here private static bool ShouldUseClrMdDisassembler(BenchmarkCase benchmarkCase) - => !ShouldUseMonoDisassembler(benchmarkCase) && (RuntimeInformation.IsWindows() || RuntimeInformation.IsLinux()); + => !ShouldUseMonoDisassembler(benchmarkCase) && (OsDetector.IsWindows() || OsDetector.IsLinux()); private static bool ShouldUseSameArchitectureDisassembler(BenchmarkCase benchmarkCase, DiagnoserActionParameters parameters) { if (ShouldUseClrMdDisassembler(benchmarkCase)) { - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) { return WindowsDisassembler.GetDisassemblerArchitecture(parameters.Process, benchmarkCase.Job.Environment.Platform) == RuntimeInformation.GetCurrentPlatform(); @@ -194,7 +196,7 @@ private class NativeCodeSizeMetricDescriptor : IMetricDescriptor public string Legend => "Native code size of the disassembled method(s)"; public string NumberFormat => "N0"; public UnitType UnitType => UnitType.CodeSize; - public string Unit => SizeUnit.B.Name; + public string Unit => SizeUnit.B.Abbreviation; public bool TheGreaterTheBetter => false; public int PriorityInCategory => 0; public bool GetIsAvailable(Metric metric) => true; diff --git a/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs index f29faae037..eb0d5ce3f6 100644 --- a/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs @@ -7,7 +7,7 @@ namespace BenchmarkDotNet.Disassemblers { - internal class IntelDisassembler : ClrMdV2Disassembler + internal class IntelDisassembler : ClrMdV3Disassembler { internal sealed class RuntimeSpecificData { @@ -46,7 +46,7 @@ internal RuntimeSpecificData(State state) } } - private static readonly Dictionary runtimeSpecificData = new (); + private static readonly Dictionary runtimeSpecificData = new(); protected override IEnumerable Decode(byte[] code, ulong startAddress, State state, int depth, ClrMethod currentMethod, DisassemblySyntax syntax) { diff --git a/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs index cf810cdd86..1a4835484d 100644 --- a/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs @@ -8,16 +8,16 @@ namespace BenchmarkDotNet.Disassemblers internal class SameArchitectureDisassembler { private readonly DisassemblyDiagnoserConfig config; - private ClrMdV2Disassembler? clrMdV2Disassembler; + private ClrMdV3Disassembler? clrMdV3Disassembler; internal SameArchitectureDisassembler(DisassemblyDiagnoserConfig config) => this.config = config; internal DisassemblyResult Disassemble(DiagnoserActionParameters parameters) // delay the creation to avoid exceptions at DisassemblyDiagnoser ctor - => (clrMdV2Disassembler ??= CreateDisassemblerForCurrentArchitecture()) + => (clrMdV3Disassembler ??= CreateDisassemblerForCurrentArchitecture()) .AttachAndDisassemble(BuildDisassemblerSettings(parameters)); - private static ClrMdV2Disassembler CreateDisassemblerForCurrentArchitecture() + private static ClrMdV3Disassembler CreateDisassemblerForCurrentArchitecture() => RuntimeInformation.GetCurrentPlatform() switch { Platform.X86 or Platform.X64 => new IntelDisassembler(), @@ -26,7 +26,7 @@ private static ClrMdV2Disassembler CreateDisassemblerForCurrentArchitecture() }; private Settings BuildDisassemblerSettings(DiagnoserActionParameters parameters) - => new ( + => new( processId: parameters.Process.Id, typeName: $"BenchmarkDotNet.Autogenerated.Runnable_{parameters.BenchmarkId.Value}", methodName: DisassemblerConstants.DisassemblerEntryMethodName, diff --git a/src/BenchmarkDotNet/Disassemblers/SourceCodeProvider.cs b/src/BenchmarkDotNet/Disassemblers/SourceCodeProvider.cs index 777c24370e..beaeebfaa7 100644 --- a/src/BenchmarkDotNet/Disassemblers/SourceCodeProvider.cs +++ b/src/BenchmarkDotNet/Disassemblers/SourceCodeProvider.cs @@ -7,14 +7,21 @@ namespace BenchmarkDotNet.Disassemblers { - internal static class SourceCodeProvider + internal class SourceCodeProvider : IDisposable { - private static readonly Dictionary SourceFileCache = new Dictionary(); - private static readonly Dictionary SourceFilePathsCache = new Dictionary(); + private readonly Dictionary sourceFileCache = new Dictionary(); + private readonly Dictionary sourceFilePathsCache = new Dictionary(); + private readonly Dictionary pdbReaders = new Dictionary(); + private readonly SymbolReader symbolReader = new SymbolReader(TextWriter.Null) { SymbolPath = SymbolPath.MicrosoftSymbolServerPath }; - internal static IEnumerable GetSource(ClrMethod method, ILToNativeMap map) + public void Dispose() { - var sourceLocation = method.GetSourceLocation(map.ILOffset); + symbolReader.Dispose(); + } + + internal IEnumerable GetSource(ClrMethod method, ILToNativeMap map) + { + var sourceLocation = GetSourceLocation(method, map.ILOffset); if (sourceLocation == null) yield break; @@ -39,12 +46,12 @@ internal static IEnumerable GetSource(ClrMethod method, ILToNativeMap map } } - private static string GetFilePath(SourceFile sourceFile) - => SourceFilePathsCache.TryGetValue(sourceFile, out string filePath) ? filePath : sourceFile.Url; + private string GetFilePath(SourceFile sourceFile) + => sourceFilePathsCache.TryGetValue(sourceFile, out string filePath) ? filePath : sourceFile.Url; - private static string ReadSourceLine(SourceFile file, int line) + private string ReadSourceLine(SourceFile file, int line) { - if (!SourceFileCache.TryGetValue(file, out string[] contents)) + if (!sourceFileCache.TryGetValue(file, out string[] contents)) { // GetSourceFile method returns path when file is stored on the same machine // otherwise it downloads it from the Symbol Server and returns the source code ;) @@ -56,14 +63,14 @@ private static string ReadSourceLine(SourceFile file, int line) if (File.Exists(wholeFileOrJustPath)) { contents = File.ReadAllLines(wholeFileOrJustPath); - SourceFilePathsCache.Add(file, wholeFileOrJustPath); + sourceFilePathsCache.Add(file, wholeFileOrJustPath); } else { contents = wholeFileOrJustPath.Split(new string[] { Environment.NewLine }, StringSplitOptions.None); } - SourceFileCache.Add(file, contents); + sourceFileCache.Add(file, contents); } return line - 1 < contents.Length @@ -99,17 +106,8 @@ private static string GetSmartPointer(string sourceLine, int? start, int? end) return new string(prefix); } - } - - internal static class ClrSourceExtensions - { - // TODO Not sure we want this to be a shared dictionary, especially without - // any synchronization. Probably want to put this hanging off the Context - // somewhere, or inside SymbolCache. - private static readonly Dictionary s_pdbReaders = new Dictionary(); - private static readonly SymbolReader symbolReader = new SymbolReader(TextWriter.Null) { SymbolPath = SymbolPath.MicrosoftSymbolServerPath }; - internal static SourceLocation GetSourceLocation(this ClrMethod method, int ilOffset) + internal SourceLocation GetSourceLocation(ClrMethod method, int ilOffset) { var reader = GetReaderForMethod(method); if (reader == null) @@ -118,7 +116,7 @@ internal static SourceLocation GetSourceLocation(this ClrMethod method, int ilOf return reader.SourceLocationForManagedCode((uint)method.MetadataToken, ilOffset); } - internal static SourceLocation GetSourceLocation(this ClrStackFrame frame) + internal SourceLocation GetSourceLocation(ClrStackFrame frame) { var reader = GetReaderForMethod(frame.Method); if (reader == null) @@ -145,7 +143,7 @@ private static int FindIlOffset(ClrStackFrame frame) return last; } - private static ManagedSymbolModule GetReaderForMethod(ClrMethod method) + private ManagedSymbolModule GetReaderForMethod(ClrMethod method) { ClrModule module = method?.Type?.Module; PdbInfo info = module?.Pdb; @@ -153,7 +151,7 @@ private static ManagedSymbolModule GetReaderForMethod(ClrMethod method) ManagedSymbolModule? reader = null; if (info != null) { - if (!s_pdbReaders.TryGetValue(info, out reader)) + if (!pdbReaders.TryGetValue(info, out reader)) { string pdbPath = info.Path; if (pdbPath != null) @@ -173,7 +171,7 @@ private static ManagedSymbolModule GetReaderForMethod(ClrMethod method) } } - s_pdbReaders[info] = reader; + pdbReaders[info] = reader; } } diff --git a/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs index 6bec70162a..8051ba3463 100644 --- a/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs @@ -7,6 +7,7 @@ using System.Text; using System.Xml; using System.Xml.Serialization; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; @@ -165,7 +166,7 @@ public static bool Is64Bit(Process process) if (Environment.GetEnvironmentVariable("PROCESSOR_ARCHITECTURE") == "x86") return false; - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) { IsWow64Process(process.Handle, out bool isWow64); diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index 29d2e0ddd7..1fc1b96dfd 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -1,10 +1,14 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.Linq; using System.Runtime.CompilerServices; +using System.Threading; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using JetBrains.Annotations; @@ -15,8 +19,6 @@ namespace BenchmarkDotNet.Engines [UsedImplicitly] public class Engine : IEngine { - public const int MinInvokeCount = 4; - [PublicAPI] public IHost Host { get; } [PublicAPI] public Action WorkloadAction { get; } [PublicAPI] public Action Dummy1Action { get; } @@ -40,10 +42,7 @@ public class Engine : IEngine private bool EvaluateOverhead { get; } private bool MemoryRandomization { get; } - private readonly List jittingMeasurements = new (10); - private readonly EnginePilotStage pilotStage; - private readonly EngineWarmupStage warmupStage; - private readonly EngineActualStage actualStage; + private readonly List jittingMeasurements = new(10); private readonly bool includeExtraStats; private readonly Random random; @@ -79,10 +78,6 @@ internal Engine( EvaluateOverhead = targetJob.ResolveValue(AccuracyMode.EvaluateOverheadCharacteristic, Resolver); MemoryRandomization = targetJob.ResolveValue(RunMode.MemoryRandomizationCharacteristic, Resolver); - warmupStage = new EngineWarmupStage(this); - pilotStage = new EnginePilotStage(this); - actualStage = new EngineActualStage(this); - random = new Random(12345); // we are using constant seed to try to get repeatable results } @@ -102,6 +97,9 @@ public void Dispose() } } + // AggressiveOptimization forces the method to go straight to tier1 JIT, and will never be re-jitted, + // eliminating tiered JIT as a potential variable in measurements. + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] public RunResults Run() { var measurements = new List(); @@ -112,29 +110,33 @@ public RunResults Run() if (EngineEventSource.Log.IsEnabled()) EngineEventSource.Log.BenchmarkStart(BenchmarkName); - if (Strategy != RunStrategy.ColdStart) + // Enumerate the stages and run iterations in a loop to ensure each benchmark invocation is called with a constant stack size. + // #1120 + foreach (var stage in EngineStage.EnumerateStages(this, Strategy, EvaluateOverhead)) { - if (Strategy != RunStrategy.Monitoring) + if (stage.Stage == IterationStage.Actual && stage.Mode == IterationMode.Workload) { - var pilotStageResult = pilotStage.Run(); - invokeCount = pilotStageResult.PerfectInvocationCount; - measurements.AddRange(pilotStageResult.Measurements); - - if (EvaluateOverhead) - { - measurements.AddRange(warmupStage.RunOverhead(invokeCount, UnrollFactor)); - measurements.AddRange(actualStage.RunOverhead(invokeCount, UnrollFactor)); - } + Host.BeforeMainRun(); } - measurements.AddRange(warmupStage.RunWorkload(invokeCount, UnrollFactor, Strategy)); - } - - Host.BeforeMainRun(); + var stageMeasurements = stage.GetMeasurementList(); + // 1-based iterationIndex + int iterationIndex = 1; + while (stage.GetShouldRunIteration(stageMeasurements, ref invokeCount)) + { + var measurement = RunIteration(new IterationData(stage.Mode, stage.Stage, iterationIndex, invokeCount, UnrollFactor)); + stageMeasurements.Add(measurement); + ++iterationIndex; + } + measurements.AddRange(stageMeasurements); - measurements.AddRange(actualStage.RunWorkload(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring)); + WriteLine(); - Host.AfterMainRun(); + if (stage.Stage == IterationStage.Actual && stage.Mode == IterationMode.Workload) + { + Host.AfterMainRun(); + } + } (GcStats workGcHasDone, ThreadingStats threadingStats, double exceptionFrequency) = includeExtraStats ? GetExtraStats(new IterationData(IterationMode.Workload, IterationStage.Actual, 0, invokeCount, UnrollFactor)) @@ -148,11 +150,15 @@ public RunResults Run() return new RunResults(measurements, outlierMode, workGcHasDone, threadingStats, exceptionFrequency); } + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] public Measurement RunIteration(IterationData data) { // Initialization long invokeCount = data.InvokeCount; int unrollFactor = data.UnrollFactor; + if (invokeCount % unrollFactor != 0) + throw new ArgumentOutOfRangeException(nameof(data), $"InvokeCount({invokeCount}) should be a multiple of UnrollFactor({unrollFactor})."); + long totalOperations = invokeCount * OperationsPerInvoke; bool isOverhead = data.IterationMode == IterationMode.Overhead; bool randomizeMemory = !isOverhead && MemoryRandomization; @@ -166,12 +172,9 @@ public Measurement RunIteration(IterationData data) if (EngineEventSource.Log.IsEnabled()) EngineEventSource.Log.IterationStart(data.IterationMode, data.IterationStage, totalOperations); - Span stackMemory = randomizeMemory ? stackalloc byte[random.Next(32)] : Span.Empty; - - // Measure - var clock = Clock.Start(); - action(invokeCount / unrollFactor); - var clockSpan = clock.GetElapsed(); + var clockSpan = randomizeMemory + ? MeasureWithRandomStack(action, invokeCount / unrollFactor) + : Measure(action, invokeCount / unrollFactor); if (EngineEventSource.Log.IsEnabled()) EngineEventSource.Log.IterationStop(data.IterationMode, data.IterationStage, totalOperations); @@ -190,42 +193,85 @@ public Measurement RunIteration(IterationData data) if (measurement.IterationStage == IterationStage.Jitting) jittingMeasurements.Add(measurement); + return measurement; + } + + // This is in a separate method, because stackalloc can affect code alignment, + // resulting in unexpected measurements on some AMD cpus, + // even if the stackalloc branch isn't executed. (#2366) + [MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)] + private unsafe ClockSpan MeasureWithRandomStack(Action action, long invokeCount) + { + byte* stackMemory = stackalloc byte[random.Next(32)]; + var clockSpan = Measure(action, invokeCount); Consume(stackMemory); + return clockSpan; + } - return measurement; + [MethodImpl(MethodImplOptions.NoInlining)] + private unsafe void Consume(byte* _) { } + + [MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)] + private ClockSpan Measure(Action action, long invokeCount) + { + var clock = Clock.Start(); + action(invokeCount); + return clock.GetElapsed(); } private (GcStats, ThreadingStats, double) GetExtraStats(IterationData data) { - // we enable monitoring after main target run, for this single iteration which is executed at the end - // so even if we enable AppDomain monitoring in separate process - // it does not matter, because we have already obtained the results! - EnableMonitoring(); + // Warm up the measurement functions before starting the actual measurement. + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(GcStats.ReadInitial()); + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(GcStats.ReadFinal()); IterationSetupAction(); // we run iteration setup first, so even if it allocates, it is not included in the results var initialThreadingStats = ThreadingStats.ReadInitial(); // this method might allocate var exceptionsStats = new ExceptionsStats(); // allocates exceptionsStats.StartListening(); // this method might allocate - var initialGcStats = GcStats.ReadInitial(); - WorkloadAction(data.InvokeCount / data.UnrollFactor); +#if !NET7_0_OR_GREATER + if (RuntimeInformation.IsNetCore && Environment.Version.Major is >= 3 and <= 6 && RuntimeInformation.IsTieredJitEnabled) + { + // #1542 + // We put the current thread to sleep so tiered jit can kick in, compile its stuff, + // and NOT allocate anything on the background thread when we are measuring allocations. + // This is only an issue on netcoreapp3.0 to net6.0. Tiered jit allocations were "fixed" in net7.0 + // (maybe not completely eliminated forever, but at least reduced to a point where measurements are much more stable), + // and netcoreapp2.X uses only GetAllocatedBytesForCurrentThread which doesn't capture the tiered jit allocations. + Thread.Sleep(TimeSpan.FromMilliseconds(500)); + } +#endif + + // GC collect before measuring allocations. + ForceGcCollect(); + GcStats gcStats; + using (FinalizerBlocker.MaybeStart()) + { + gcStats = MeasureWithGc(data.InvokeCount / data.UnrollFactor); + } - exceptionsStats.Stop(); - var finalGcStats = GcStats.ReadFinal(); + exceptionsStats.Stop(); // this method might (de)allocate var finalThreadingStats = ThreadingStats.ReadFinal(); IterationCleanupAction(); // we run iteration cleanup after collecting GC stats var totalOperationsCount = data.InvokeCount * OperationsPerInvoke; - GcStats gcStats = (finalGcStats - initialGcStats).WithTotalOperations(totalOperationsCount); - ThreadingStats threadingStats = (finalThreadingStats - initialThreadingStats).WithTotalOperations(data.InvokeCount * OperationsPerInvoke); - - return (gcStats, threadingStats, exceptionsStats.ExceptionsCount / (double)totalOperationsCount); + return (gcStats.WithTotalOperations(totalOperationsCount), + (finalThreadingStats - initialThreadingStats).WithTotalOperations(totalOperationsCount), + exceptionsStats.ExceptionsCount / (double)totalOperationsCount); } - [MethodImpl(MethodImplOptions.NoInlining)] - private void Consume(in Span _) { } + // Isolate the allocation measurement and skip tier0 jit to make sure we don't get any unexpected allocations. + [MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)] + private GcStats MeasureWithGc(long invokeCount) + { + var initialGcStats = GcStats.ReadInitial(); + WorkloadAction(invokeCount); + var finalGcStats = GcStats.ReadFinal(); + return finalGcStats - initialGcStats; + } private void RandomizeManagedHeapMemory() { @@ -253,7 +299,7 @@ private void GcCollect() ForceGcCollect(); } - private static void ForceGcCollect() + internal static void ForceGcCollect() { GC.Collect(); GC.WaitForPendingFinalizers(); @@ -264,15 +310,6 @@ private static void ForceGcCollect() public void WriteLine() => Host.WriteLine(); - private static void EnableMonitoring() - { - if (RuntimeInformation.IsOldMono) // Monitoring is not available in Mono, see http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes-in-mono - return; - - if (RuntimeInformation.IsFullFramework) - AppDomain.MonitoringIsEnabled = true; - } - [UsedImplicitly] public static class Signals { @@ -295,5 +332,71 @@ private static readonly Dictionary MessagesToSignals public static bool TryGetSignal(string message, out HostSignal signal) => MessagesToSignals.TryGetValue(message, out signal); } + + // Very long key and value so this shouldn't be used outside of unit tests. + internal const string UnitTestBlockFinalizerEnvKey = "BENCHMARKDOTNET_UNITTEST_BLOCK_FINALIZER_FOR_MEMORYDIAGNOSER"; + internal const string UnitTestBlockFinalizerEnvValue = UnitTestBlockFinalizerEnvKey + "_ACTIVE"; + + // To prevent finalizers interfering with allocation measurements for unit tests, + // we block the finalizer thread until we've completed the measurement. + // https://github.com/dotnet/runtime/issues/101536#issuecomment-2077647417 + private readonly struct FinalizerBlocker : IDisposable + { + private readonly object hangLock; + + private FinalizerBlocker(object hangLock) => this.hangLock = hangLock; + + private sealed class Impl + { + // ManualResetEvent(Slim) allocates when it is waited and yields the thread, + // so we use Monitor.Wait instead which does not allocate managed memory. + // This behavior is not documented, but was observed with the VS Profiler. + private readonly object hangLock = new(); + private readonly ManualResetEventSlim enteredFinalizerEvent = new(false); + + ~Impl() + { + lock (hangLock) + { + enteredFinalizerEvent.Set(); + Monitor.Wait(hangLock); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + internal static (object hangLock, ManualResetEventSlim enteredFinalizerEvent) CreateWeakly() + { + var impl = new Impl(); + return (impl.hangLock, impl.enteredFinalizerEvent); + } + } + + internal static FinalizerBlocker MaybeStart() + { + if (Environment.GetEnvironmentVariable(UnitTestBlockFinalizerEnvKey) != UnitTestBlockFinalizerEnvValue) + { + return default; + } + var (hangLock, enteredFinalizerEvent) = Impl.CreateWeakly(); + do + { + GC.Collect(); + // Do NOT call GC.WaitForPendingFinalizers. + } + while (!enteredFinalizerEvent.IsSet); + return new FinalizerBlocker(hangLock); + } + + public void Dispose() + { + if (hangLock is not null) + { + lock (hangLock) + { + Monitor.Pulse(hangLock); + } + } + } + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EngineActualStage.cs b/src/BenchmarkDotNet/Engines/EngineActualStage.cs new file mode 100644 index 0000000000..173f4e3432 --- /dev/null +++ b/src/BenchmarkDotNet/Engines/EngineActualStage.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Mathematics; +using BenchmarkDotNet.Reports; +using Perfolizer.Horology; +using Perfolizer.Mathematics.OutlierDetection; + +namespace BenchmarkDotNet.Engines +{ + internal abstract class EngineActualStage(IterationMode iterationMode) : EngineStage(IterationStage.Actual, iterationMode) + { + internal const int MaxOverheadIterationCount = 20; + + internal static EngineActualStage GetOverhead(IEngine engine) + => new EngineActualStageAuto(engine.TargetJob, engine.Resolver, IterationMode.Overhead); + + internal static EngineActualStage GetWorkload(IEngine engine, RunStrategy strategy) + { + var targetJob = engine.TargetJob; + int? iterationCount = targetJob.ResolveValueAsNullable(RunMode.IterationCountCharacteristic); + const int DefaultWorkloadCount = 10; + return iterationCount == null && strategy != RunStrategy.Monitoring + ? new EngineActualStageAuto(targetJob, engine.Resolver, IterationMode.Workload) + : new EngineActualStageSpecific(iterationCount ?? DefaultWorkloadCount, IterationMode.Workload); + } + } + + internal sealed class EngineActualStageAuto : EngineActualStage + { + private readonly double maxRelativeError; + private readonly TimeInterval? maxAbsoluteError; + private readonly OutlierMode outlierMode; + private readonly int minIterationCount; + private readonly int maxIterationCount; + private readonly List measurementsForStatistics; + private int iterationCounter = 0; + + public EngineActualStageAuto(Job targetJob, IResolver resolver, IterationMode iterationMode) : base(iterationMode) + { + maxRelativeError = targetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, resolver); + maxAbsoluteError = targetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); + outlierMode = targetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, resolver); + minIterationCount = targetJob.ResolveValue(RunMode.MinIterationCountCharacteristic, resolver); + maxIterationCount = targetJob.ResolveValue(RunMode.MaxIterationCountCharacteristic, resolver); + measurementsForStatistics = GetMeasurementList(); + } + + internal override List GetMeasurementList() => new(maxIterationCount); + + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) + { + if (measurements.Count == 0) + { + return true; + } + + const double MaxOverheadRelativeError = 0.05; + bool isOverhead = Mode == IterationMode.Overhead; + double effectiveMaxRelativeError = isOverhead ? MaxOverheadRelativeError : maxRelativeError; + iterationCounter++; + var measurement = measurements[measurements.Count - 1]; + measurementsForStatistics.Add(measurement); + + var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, outlierMode); + double actualError = statistics.LegacyConfidenceInterval.Margin; + + double maxError1 = effectiveMaxRelativeError * statistics.Mean; + double maxError2 = maxAbsoluteError?.Nanoseconds ?? double.MaxValue; + double maxError = Math.Min(maxError1, maxError2); + + if (iterationCounter >= minIterationCount && actualError < maxError) + { + return false; + } + + if (iterationCounter >= maxIterationCount || isOverhead && iterationCounter >= MaxOverheadIterationCount) + { + return false; + } + + return true; + } + } + + internal sealed class EngineActualStageSpecific(int maxIterationCount, IterationMode iterationMode) : EngineActualStage(iterationMode) + { + private int iterationCount = 0; + + internal override List GetMeasurementList() => new(maxIterationCount); + + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) + => ++iterationCount <= maxIterationCount; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs b/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs deleted file mode 100644 index cecd779899..0000000000 --- a/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs +++ /dev/null @@ -1,90 +0,0 @@ -using System; -using System.Collections.Generic; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Mathematics; -using BenchmarkDotNet.Reports; -using Perfolizer.Horology; -using Perfolizer.Mathematics.OutlierDetection; - -namespace BenchmarkDotNet.Engines -{ - public class EngineActualStage : EngineStage - { - internal const int MaxOverheadIterationCount = 20; - private const double MaxOverheadRelativeError = 0.05; - private const int DefaultWorkloadCount = 10; - - private readonly int? iterationCount; - private readonly double maxRelativeError; - private readonly TimeInterval? maxAbsoluteError; - private readonly OutlierMode outlierMode; - private readonly int minIterationCount; - private readonly int maxIterationCount; - - public EngineActualStage(IEngine engine) : base(engine) - { - iterationCount = engine.TargetJob.ResolveValueAsNullable(RunMode.IterationCountCharacteristic); - maxRelativeError = engine.TargetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, engine.Resolver); - maxAbsoluteError = engine.TargetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); - outlierMode = engine.TargetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, engine.Resolver); - minIterationCount = engine.TargetJob.ResolveValue(RunMode.MinIterationCountCharacteristic, engine.Resolver); - maxIterationCount = engine.TargetJob.ResolveValue(RunMode.MaxIterationCountCharacteristic, engine.Resolver); - } - - public IReadOnlyList RunOverhead(long invokeCount, int unrollFactor) - => RunAuto(invokeCount, IterationMode.Overhead, unrollFactor); - - public IReadOnlyList RunWorkload(long invokeCount, int unrollFactor, bool forceSpecific = false) - => Run(invokeCount, IterationMode.Workload, false, unrollFactor, forceSpecific); - - internal IReadOnlyList Run(long invokeCount, IterationMode iterationMode, bool runAuto, int unrollFactor, bool forceSpecific = false) - => (runAuto || iterationCount == null) && !forceSpecific - ? RunAuto(invokeCount, iterationMode, unrollFactor) - : RunSpecific(invokeCount, iterationMode, iterationCount ?? DefaultWorkloadCount, unrollFactor); - - private List RunAuto(long invokeCount, IterationMode iterationMode, int unrollFactor) - { - var measurements = new List(maxIterationCount); - var measurementsForStatistics = new List(maxIterationCount); - - int iterationCounter = 0; - bool isOverhead = iterationMode == IterationMode.Overhead; - double effectiveMaxRelativeError = isOverhead ? MaxOverheadRelativeError : maxRelativeError; - while (true) - { - iterationCounter++; - var measurement = RunIteration(iterationMode, IterationStage.Actual, iterationCounter, invokeCount, unrollFactor); - measurements.Add(measurement); - measurementsForStatistics.Add(measurement); - - var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, outlierMode); - double actualError = statistics.ConfidenceInterval.Margin; - - double maxError1 = effectiveMaxRelativeError * statistics.Mean; - double maxError2 = maxAbsoluteError?.Nanoseconds ?? double.MaxValue; - double maxError = Math.Min(maxError1, maxError2); - - if (iterationCounter >= minIterationCount && actualError < maxError) - break; - - if (iterationCounter >= maxIterationCount || isOverhead && iterationCounter >= MaxOverheadIterationCount) - break; - } - WriteLine(); - - return measurements; - } - - private List RunSpecific(long invokeCount, IterationMode iterationMode, int iterationCount, int unrollFactor) - { - var measurements = new List(iterationCount); - - for (int i = 0; i < iterationCount; i++) - measurements.Add(RunIteration(iterationMode, IterationStage.Actual, i + 1, invokeCount, unrollFactor)); - - WriteLine(); - - return measurements; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EnginePilotStage.cs b/src/BenchmarkDotNet/Engines/EnginePilotStage.cs index 6adf6d964f..43b543d5c1 100644 --- a/src/BenchmarkDotNet/Engines/EnginePilotStage.cs +++ b/src/BenchmarkDotNet/Engines/EnginePilotStage.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; @@ -8,130 +9,105 @@ namespace BenchmarkDotNet.Engines { // TODO: use clockResolution - internal class EnginePilotStage : EngineStage + internal abstract class EnginePilotStage(Job targetJob, IResolver resolver) : EngineStage(IterationStage.Pilot, IterationMode.Workload) { - public readonly struct PilotStageResult - { - public long PerfectInvocationCount { get; } - public IReadOnlyList Measurements { get; } + internal const long MaxInvokeCount = (long.MaxValue / 2 + 1) / 2; - public PilotStageResult(long perfectInvocationCount, List measurements) - { - PerfectInvocationCount = perfectInvocationCount; - Measurements = measurements; - } + protected readonly int unrollFactor = targetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, resolver); + protected readonly int minInvokeCount = targetJob.ResolveValue(AccuracyMode.MinInvokeCountCharacteristic, resolver); - public PilotStageResult(long perfectInvocationCount) - { - PerfectInvocationCount = perfectInvocationCount; - Measurements = Array.Empty(); - } + protected long Autocorrect(long count) => (count + unrollFactor - 1) / unrollFactor * unrollFactor; + + internal static EnginePilotStage GetStage(IEngine engine) + { + var targetJob = engine.TargetJob; + // If InvocationCount is specified, pilot stage should be skipped + return targetJob.HasValue(RunMode.InvocationCountCharacteristic) ? null + // Here we want to guess "perfect" amount of invocation + : targetJob.HasValue(RunMode.IterationTimeCharacteristic) ? new EnginePilotStageSpecific(targetJob, engine.Resolver) + : new EnginePilotStageAuto(targetJob, engine.Resolver); } + } - internal const long MaxInvokeCount = (long.MaxValue / 2 + 1) / 2; + internal sealed class EnginePilotStageAuto(Job targetJob, IResolver resolver) : EnginePilotStage(targetJob, resolver) + { + private readonly TimeInterval minIterationTime = targetJob.ResolveValue(AccuracyMode.MinIterationTimeCharacteristic, resolver); + private readonly double maxRelativeError = targetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, resolver); + private readonly TimeInterval? maxAbsoluteError = targetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); + private readonly double resolution = targetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, resolver).GetResolution().Nanoseconds; - private readonly int unrollFactor; - private readonly TimeInterval minIterationTime; - private readonly int minInvokeCount; - private readonly double maxRelativeError; - private readonly TimeInterval? maxAbsoluteError; - private readonly double targetIterationTime; - private readonly double resolution; + internal override List GetMeasurementList() => []; - public EnginePilotStage(IEngine engine) : base(engine) + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) { - unrollFactor = engine.TargetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, engine.Resolver); - minIterationTime = engine.TargetJob.ResolveValue(AccuracyMode.MinIterationTimeCharacteristic, engine.Resolver); - minInvokeCount = engine.TargetJob.ResolveValue(AccuracyMode.MinInvokeCountCharacteristic, engine.Resolver); - maxRelativeError = engine.TargetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, engine.Resolver); - maxAbsoluteError = engine.TargetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); - targetIterationTime = engine.TargetJob.ResolveValue(RunMode.IterationTimeCharacteristic, engine.Resolver).ToNanoseconds(); - resolution = engine.TargetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, engine.Resolver).GetResolution().Nanoseconds; - } + if (measurements.Count == 0) + { + invokeCount = Autocorrect(minInvokeCount); + return true; + } - /// Perfect invocation count - public PilotStageResult Run() - { - // If InvocationCount is specified, pilot stage should be skipped - if (TargetJob.HasValue(RunMode.InvocationCountCharacteristic)) - return new PilotStageResult(TargetJob.Run.InvocationCount); + var measurement = measurements[measurements.Count - 1]; + double iterationTime = measurement.Nanoseconds; + double operationError = 2.0 * resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision - // Here we want to guess "perfect" amount of invocation - return TargetJob.HasValue(RunMode.IterationTimeCharacteristic) - ? RunSpecific() - : RunAuto(); - } + // Max acceptable operation error + double operationMaxError1 = iterationTime / invokeCount * maxRelativeError; + double operationMaxError2 = maxAbsoluteError?.Nanoseconds ?? double.MaxValue; + double operationMaxError = Math.Min(operationMaxError1, operationMaxError2); - /// - /// A case where we don't have specific iteration time. - /// - private PilotStageResult RunAuto() - { - long invokeCount = Autocorrect(minInvokeCount); - var measurements = new List(); + bool isFinished = operationError < operationMaxError && iterationTime >= minIterationTime.Nanoseconds; + if (isFinished || invokeCount >= MaxInvokeCount) + { + return false; + } - int iterationCounter = 0; - while (true) + if (unrollFactor == 1 && invokeCount < EnvironmentResolver.DefaultUnrollFactorForThroughput) { - iterationCounter++; - var measurement = RunIteration(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, unrollFactor); - measurements.Add(measurement); - double iterationTime = measurement.Nanoseconds; - double operationError = 2.0 * resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision - - // Max acceptable operation error - double operationMaxError1 = iterationTime / invokeCount * maxRelativeError; - double operationMaxError2 = maxAbsoluteError?.Nanoseconds ?? double.MaxValue; - double operationMaxError = Math.Min(operationMaxError1, operationMaxError2); - - bool isFinished = operationError < operationMaxError && iterationTime >= minIterationTime.Nanoseconds; - if (isFinished) - break; - if (invokeCount >= MaxInvokeCount) - break; - - if (unrollFactor == 1 && invokeCount < EnvironmentResolver.DefaultUnrollFactorForThroughput) - invokeCount += 1; - else - invokeCount *= 2; + ++invokeCount; + } + else + { + invokeCount *= 2; } - WriteLine(); - return new PilotStageResult(invokeCount, measurements); + return true; } + } - /// - /// A case where we have specific iteration time. - /// - private PilotStageResult RunSpecific() - { - long invokeCount = Autocorrect(Engine.MinInvokeCount); - var measurements = new List(); + internal sealed class EnginePilotStageSpecific(Job targetJob, IResolver resolver) : EnginePilotStage(targetJob, resolver) + { + private const int MinInvokeCount = 4; - int iterationCounter = 0; + private readonly double targetIterationTime = targetJob.ResolveValue(RunMode.IterationTimeCharacteristic, resolver).ToNanoseconds(); - int downCount = 0; // Amount of iterations where newInvokeCount < invokeCount - while (true) + private int _downCount = 0; // Amount of iterations where newInvokeCount < invokeCount + + internal override List GetMeasurementList() => []; + + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) + { + if (measurements.Count == 0) { - iterationCounter++; - var measurement = RunIteration(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, unrollFactor); - measurements.Add(measurement); - double actualIterationTime = measurement.Nanoseconds; - long newInvokeCount = Autocorrect(Math.Max(minInvokeCount, (long)Math.Round(invokeCount * targetIterationTime / actualIterationTime))); + invokeCount = Autocorrect(MinInvokeCount); + return true; + } - if (newInvokeCount < invokeCount) - downCount++; + var measurement = measurements[measurements.Count - 1]; + double actualIterationTime = measurement.Nanoseconds; + long newInvokeCount = Autocorrect(Math.Max(minInvokeCount, (long) Math.Round(invokeCount * targetIterationTime / actualIterationTime))); - if (Math.Abs(newInvokeCount - invokeCount) <= 1 || downCount >= 3) - break; + if (newInvokeCount < invokeCount) + { + _downCount++; + } - invokeCount = newInvokeCount; + if (Math.Abs(newInvokeCount - invokeCount) <= 1 || _downCount >= 3) + { + return false; } - WriteLine(); - return new PilotStageResult(invokeCount, measurements); + invokeCount = newInvokeCount; + return true; } - - private long Autocorrect(long count) => (count + unrollFactor - 1) / unrollFactor * unrollFactor; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EngineStage.cs b/src/BenchmarkDotNet/Engines/EngineStage.cs index 2c3b825394..82b6bc5fb3 100644 --- a/src/BenchmarkDotNet/Engines/EngineStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineStage.cs @@ -1,48 +1,43 @@ -using System; -using System.Collections.Generic; -using BenchmarkDotNet.Jobs; +using System.Collections.Generic; +using System.Runtime.CompilerServices; using BenchmarkDotNet.Reports; namespace BenchmarkDotNet.Engines { - public class EngineStage + internal abstract class EngineStage(IterationStage stage, IterationMode mode) { - private readonly IEngine engine; + internal readonly IterationStage Stage = stage; + internal readonly IterationMode Mode = mode; - protected EngineStage(IEngine engine) => this.engine = engine; + internal abstract List GetMeasurementList(); + internal abstract bool GetShouldRunIteration(List measurements, ref long invokeCount); - protected Job TargetJob => engine.TargetJob; - - protected Measurement RunIteration(IterationMode mode, IterationStage stage, int index, long invokeCount, int unrollFactor) - { - if (invokeCount % unrollFactor != 0) - throw new ArgumentOutOfRangeException($"InvokeCount({invokeCount}) should be a multiple of UnrollFactor({unrollFactor})."); - return engine.RunIteration(new IterationData(mode, stage, index, invokeCount, unrollFactor)); - } - - internal List Run(IStoppingCriteria criteria, long invokeCount, IterationMode mode, IterationStage stage, int unrollFactor) + [MethodImpl(MethodImplOptions.NoInlining)] + internal static IEnumerable EnumerateStages(IEngine engine, RunStrategy strategy, bool evaluateOverhead) { - var measurements = new List(criteria.MaxIterationCount); - if (criteria.Evaluate(measurements).IsFinished) - { - WriteLine(); - return measurements; - } + // It might be possible to add the jitting stage to this, but it's done in EngineFactory.CreateReadyToRun for now. - int iterationCounter = 0; - while (true) + if (strategy != RunStrategy.ColdStart) { - iterationCounter++; - measurements.Add(RunIteration(mode, stage, iterationCounter, invokeCount, unrollFactor)); - if (criteria.Evaluate(measurements).IsFinished) - break; + if (strategy != RunStrategy.Monitoring) + { + var pilotStage = EnginePilotStage.GetStage(engine); + if (pilotStage != null) + { + yield return pilotStage; + } + + if (evaluateOverhead) + { + yield return EngineWarmupStage.GetOverhead(); + yield return EngineActualStage.GetOverhead(engine); + } + } + + yield return EngineWarmupStage.GetWorkload(engine, strategy); } - WriteLine(); - - return measurements; + yield return EngineActualStage.GetWorkload(engine, strategy); } - - protected void WriteLine() => engine.WriteLine(); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs b/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs index 08ec7ec729..822ea69101 100644 --- a/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs @@ -1,24 +1,78 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; namespace BenchmarkDotNet.Engines { - internal class EngineWarmupStage : EngineStage + internal abstract class EngineWarmupStage(IterationMode iterationMode) : EngineStage(IterationStage.Warmup, iterationMode) { - private readonly IEngine engine; + private const int MinOverheadIterationCount = 4; + internal const int MaxOverheadIterationCount = 10; - public EngineWarmupStage(IEngine engine) : base(engine) => this.engine = engine; + internal static EngineWarmupStage GetOverhead() + => new EngineWarmupStageAuto(IterationMode.Overhead, MinOverheadIterationCount, MaxOverheadIterationCount); - public IReadOnlyList RunOverhead(long invokeCount, int unrollFactor) - => Run(invokeCount, IterationMode.Overhead, unrollFactor, RunStrategy.Throughput); + internal static EngineWarmupStage GetWorkload(IEngine engine, RunStrategy runStrategy) + { + var job = engine.TargetJob; + var count = job.ResolveValueAsNullable(RunMode.WarmupCountCharacteristic); + if (count.HasValue && count.Value != EngineResolver.ForceAutoWarmup || runStrategy == RunStrategy.Monitoring) + { + return new EngineWarmupStageSpecific(count ?? 0, IterationMode.Workload); + } + + int minIterationCount = job.ResolveValue(RunMode.MinWarmupIterationCountCharacteristic, engine.Resolver); + int maxIterationCount = job.ResolveValue(RunMode.MaxWarmupIterationCountCharacteristic, engine.Resolver); + return new EngineWarmupStageAuto(IterationMode.Workload, minIterationCount, maxIterationCount); + } + } - public IReadOnlyList RunWorkload(long invokeCount, int unrollFactor, RunStrategy runStrategy) - => Run(invokeCount, IterationMode.Workload, unrollFactor, runStrategy); + internal sealed class EngineWarmupStageAuto(IterationMode iterationMode, int minIterationCount, int maxIterationCount) : EngineWarmupStage(iterationMode) + { + private const int MinFluctuationCount = 4; + + private readonly int minIterationCount = minIterationCount; + private readonly int maxIterationCount = maxIterationCount; - internal IReadOnlyList Run(long invokeCount, IterationMode iterationMode, int unrollFactor, RunStrategy runStrategy) + internal override List GetMeasurementList() => new(maxIterationCount); + + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) { - var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(engine.TargetJob, engine.Resolver, iterationMode, runStrategy); - return Run(criteria, invokeCount, iterationMode, IterationStage.Warmup, unrollFactor); + int n = measurements.Count; + + if (n >= maxIterationCount) + { + return false; + } + if (n < minIterationCount) + { + return true; + } + + int direction = -1; // The default "pre-state" is "decrease mode" + int fluctuationCount = 0; + for (int i = 1; i < n; i++) + { + int nextDirection = Math.Sign(measurements[i].Nanoseconds - measurements[i - 1].Nanoseconds); + if (nextDirection != direction || nextDirection == 0) + { + direction = nextDirection; + fluctuationCount++; + } + } + + return fluctuationCount < MinFluctuationCount; } } + + internal sealed class EngineWarmupStageSpecific(int maxIterationCount, IterationMode iterationMode) : EngineWarmupStage(iterationMode) + { + private int iterationCount = 0; + + internal override List GetMeasurementList() => new(maxIterationCount); + + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) + => ++iterationCount <= maxIterationCount; + } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/GcStats.cs b/src/BenchmarkDotNet/Engines/GcStats.cs index a156c88161..3416d14ea4 100644 --- a/src/BenchmarkDotNet/Engines/GcStats.cs +++ b/src/BenchmarkDotNet/Engines/GcStats.cs @@ -1,5 +1,6 @@ using System; using System.Reflection; +using System.Runtime.CompilerServices; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -14,11 +15,6 @@ public struct GcStats : IEquatable public static readonly long AllocationQuantum = CalculateAllocationQuantumSize(); -#if !NET6_0_OR_GREATER - private static readonly Func GetAllocatedBytesForCurrentThreadDelegate = CreateGetAllocatedBytesForCurrentThreadDelegate(); - private static readonly Func GetTotalAllocatedBytesDelegate = CreateGetTotalAllocatedBytesDelegate(); -#endif - public static readonly GcStats Empty = default; private GcStats(int gen0Collections, int gen1Collections, int gen2Collections, long? allocatedBytes, long totalOperations) @@ -111,9 +107,10 @@ public int GetCollectionsCount(int generation) return AllocatedBytes <= AllocationQuantum ? 0L : AllocatedBytes; } + // Skip tier0 jit to make sure we don't get any unexpected allocations in this method. + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] public static GcStats ReadInitial() { - // this will force GC.Collect, so we want to do this before collecting collections counts long? allocatedBytes = GetAllocatedBytes(); return new GcStats( @@ -124,15 +121,14 @@ public static GcStats ReadInitial() 0); } + // Skip tier0 jit to make sure we don't get any unexpected allocations in this method. + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] public static GcStats ReadFinal() { return new GcStats( GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2), - - // this will force GC.Collect, so we want to do this after collecting collections counts - // to exclude this single full forced collection from results GetAllocatedBytes(), 0); } @@ -141,50 +137,30 @@ public static GcStats ReadFinal() public static GcStats FromForced(int forcedFullGarbageCollections) => new GcStats(forcedFullGarbageCollections, forcedFullGarbageCollections, forcedFullGarbageCollections, 0, 0); + // Skip tier0 jit to make sure we don't get any unexpected allocations in this method. + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] private static long? GetAllocatedBytes() { - if (RuntimeInformation.IsOldMono) // Monitoring is not available in Mono, see http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes- - return null; - // we have no tests for WASM and don't want to risk introducing a new bug (https://github.com/dotnet/BenchmarkDotNet/issues/2226) if (RuntimeInformation.IsWasm) return null; - // "This instance Int64 property returns the number of bytes that have been allocated by a specific - // AppDomain. The number is accurate as of the last garbage collection." - CLR via C# - // so we enforce GC.Collect here just to make sure we get accurate results - GC.Collect(); - - if (RuntimeInformation.IsFullFramework) // it can be a .NET app consuming our .NET Standard package - return AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize; - + // Do NOT call GC.Collect() here, as it causes finalizers to run and possibly allocate. https://github.com/dotnet/runtime/issues/101536#issuecomment-2077533242 + // Instead, we call it before we start the measurement in the Engine. #if NET6_0_OR_GREATER return GC.GetTotalAllocatedBytes(precise: true); #else - if (GetTotalAllocatedBytesDelegate != null) // it's .NET Core 3.0 with the new API available - return GetTotalAllocatedBytesDelegate.Invoke(true); // true for the "precise" argument - - // https://apisof.net/catalog/System.GC.GetAllocatedBytesForCurrentThread() is not part of the .NET Standard, so we use reflection to call it.. - return GetAllocatedBytesForCurrentThreadDelegate.Invoke(); -#endif - } - - private static Func CreateGetAllocatedBytesForCurrentThreadDelegate() - { - // this method is not a part of .NET Standard so we need to use reflection - var method = typeof(GC).GetTypeInfo().GetMethod("GetAllocatedBytesForCurrentThread", BindingFlags.Public | BindingFlags.Static); + if (GcHelpers.GetTotalAllocatedBytesDelegate != null) // it's .NET Core 3.0 with the new API available + return GcHelpers.GetTotalAllocatedBytesDelegate.Invoke(true); // true for the "precise" argument - // we create delegate to avoid boxing, IMPORTANT! - return method != null ? (Func)method.CreateDelegate(typeof(Func)) : null; - } + if (GcHelpers.CanUseMonitoringTotalAllocatedMemorySize) // Monitoring is not available in Mono, see http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes- + return AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize; - private static Func CreateGetTotalAllocatedBytesDelegate() - { - // this method is not a part of .NET Standard so we need to use reflection - var method = typeof(GC).GetTypeInfo().GetMethod("GetTotalAllocatedBytes", BindingFlags.Public | BindingFlags.Static); + if (GcHelpers.GetAllocatedBytesForCurrentThreadDelegate != null) + return GcHelpers.GetAllocatedBytesForCurrentThreadDelegate.Invoke(); - // we create delegate to avoid boxing, IMPORTANT! - return method != null ? (Func)method.CreateDelegate(typeof(Func)) : null; + return null; +#endif } public string ToOutputLine() @@ -242,9 +218,7 @@ private static long CalculateAllocationQuantumSize() break; } - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); + Engine.ForceGcCollect(); result = GC.GetTotalMemory(false); var tmp = new object(); @@ -260,5 +234,81 @@ private static long CalculateAllocationQuantumSize() public override bool Equals(object obj) => obj is GcStats other && Equals(other); public override int GetHashCode() => HashCode.Combine(Gen0Collections, Gen1Collections, Gen2Collections, AllocatedBytes, TotalOperations); + +#if !NET6_0_OR_GREATER + // Separate class to have the cctor run lazily, to avoid enabling monitoring before the benchmarks are ran. + private static class GcHelpers + { + // do not reorder these, CheckMonitoringTotalAllocatedMemorySize relies on GetTotalAllocatedBytesDelegate being initialized first + public static readonly Func GetTotalAllocatedBytesDelegate = CreateGetTotalAllocatedBytesDelegate(); + public static readonly Func GetAllocatedBytesForCurrentThreadDelegate = CreateGetAllocatedBytesForCurrentThreadDelegate(); + public static readonly bool CanUseMonitoringTotalAllocatedMemorySize = CheckMonitoringTotalAllocatedMemorySize(); + + private static Func CreateGetTotalAllocatedBytesDelegate() + { + try + { + // this method is not a part of .NET Standard so we need to use reflection + var method = typeof(GC).GetTypeInfo().GetMethod("GetTotalAllocatedBytes", BindingFlags.Public | BindingFlags.Static); + + if (method == null) + return null; + + // we create delegate to avoid boxing, IMPORTANT! + var del = (Func)method.CreateDelegate(typeof(Func)); + + // verify the api works + return del.Invoke(true) >= 0 ? del : null; + } + catch + { + return null; + } + } + + private static Func CreateGetAllocatedBytesForCurrentThreadDelegate() + { + try + { + // this method is not a part of .NET Standard so we need to use reflection + var method = typeof(GC).GetTypeInfo().GetMethod("GetAllocatedBytesForCurrentThread", BindingFlags.Public | BindingFlags.Static); + + if (method == null) + return null; + + // we create delegate to avoid boxing, IMPORTANT! + var del = (Func)method.CreateDelegate(typeof(Func)); + + // verify the api works + return del.Invoke() >= 0 ? del : null; + } + catch + { + return null; + } + } + + private static bool CheckMonitoringTotalAllocatedMemorySize() + { + try + { + // we potentially don't want to enable monitoring if we don't need it + if (GetTotalAllocatedBytesDelegate != null) + return false; + + // check if monitoring is enabled + if (!AppDomain.MonitoringIsEnabled) + AppDomain.MonitoringIsEnabled = true; + + // verify the api works + return AppDomain.MonitoringIsEnabled && AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize >= 0; + } + catch + { + return false; + } + } + } +#endif } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Engines/HostSignal.cs b/src/BenchmarkDotNet/Engines/HostSignal.cs index f4a2f75a22..cf423d2268 100644 --- a/src/BenchmarkDotNet/Engines/HostSignal.cs +++ b/src/BenchmarkDotNet/Engines/HostSignal.cs @@ -7,6 +7,11 @@ public enum HostSignal /// BeforeProcessStart, + /// + /// right after we start the benchmarking process + /// + AfterProcessStart, + /// /// before jitting, warmup /// diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/AutoWarmupStoppingCriteria.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/AutoWarmupStoppingCriteria.cs deleted file mode 100644 index e2282c7baa..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/AutoWarmupStoppingCriteria.cs +++ /dev/null @@ -1,97 +0,0 @@ -using System; -using System.Collections.Generic; -using BenchmarkDotNet.Reports; - -namespace BenchmarkDotNet.Engines -{ - /// - /// Automatically choose the best number of iterations during the warmup stage. - /// - public class AutoWarmupStoppingCriteria : StoppingCriteriaBase - { - private const int DefaultMinFluctuationCount = 4; - - private readonly int minIterationCount; - private readonly int maxIterationCount; - private readonly int minFluctuationCount; - - private readonly string maxIterationMessage; - private readonly string minFluctuationMessage; - - /// - /// The idea of the implementation is simple: if measurements monotonously decrease or increase, the steady state is not achieved; - /// we should continue the warmup stage. The evaluation method counts "fluctuations" - /// (3 consecutive measurements A,B,C where (A>B AND B<C) OR (A<B AND B>C)) until the required amount of flotations is observed. - /// - /// - /// The minimum number of iterations. - /// We are always going to do at least iterations regardless of fluctuations. - /// - /// - /// The maximum number of iterations. - /// We are always going to do at most iterations regardless of fluctuations. - /// - /// - /// The required number of fluctuations. - /// If the required number of fluctuations is achieved but the number of iterations less than , - /// we need more iterations. - /// If the required number of fluctuations is not achieved but the number of iterations equal to , - /// we should stop the iterations. - /// - public AutoWarmupStoppingCriteria(int minIterationCount, int maxIterationCount, int minFluctuationCount = DefaultMinFluctuationCount) - { - this.minIterationCount = minIterationCount; - this.maxIterationCount = maxIterationCount; - this.minFluctuationCount = minFluctuationCount; - - maxIterationMessage = $"The maximum amount of iteration ({maxIterationCount}) is achieved"; - minFluctuationMessage = $"The minimum amount of fluctuation ({minFluctuationCount}) and " + - $"the minimum amount of iterations ({minIterationCount}) are achieved"; - } - - public override StoppingResult Evaluate(IReadOnlyList measurements) - { - int n = measurements.Count; - - if (n >= maxIterationCount) - return StoppingResult.CreateFinished(maxIterationMessage); - if (n < minIterationCount) - return StoppingResult.NotFinished; - - int direction = -1; // The default "pre-state" is "decrease mode" - int fluctuationCount = 0; - for (int i = 1; i < n; i++) - { - int nextDirection = Math.Sign(measurements[i].Nanoseconds - measurements[i - 1].Nanoseconds); - if (nextDirection != direction || nextDirection == 0) - { - direction = nextDirection; - fluctuationCount++; - } - } - - return fluctuationCount >= minFluctuationCount - ? StoppingResult.CreateFinished(minFluctuationMessage) - : StoppingResult.NotFinished; - } - - protected override string GetTitle() => $"{nameof(AutoWarmupStoppingCriteria)}(" + - $"{nameof(minIterationCount)}={minIterationCount}, " + - $"{nameof(maxIterationCount)}={maxIterationCount}, " + - $"{nameof(minFluctuationCount)}={minFluctuationCount})"; - - protected override int GetMaxIterationCount() => maxIterationCount; - - protected override IEnumerable GetWarnings() - { - if (minIterationCount < 0) - yield return $"Min Iteration Count ({minIterationCount}) is negative"; - if (maxIterationCount < 0) - yield return $"Max Iteration Count ({maxIterationCount}) is negative"; - if (minFluctuationCount < 0) - yield return $"Min Fluctuation Count ({minFluctuationCount}) is negative"; - if (minIterationCount > maxIterationCount) - yield return $"Min Iteration Count ({minFluctuationCount}) is greater than Max Iteration Count ({maxIterationCount})"; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/DefaultStoppingCriteriaFactory.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/DefaultStoppingCriteriaFactory.cs deleted file mode 100644 index 356fc7362e..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/DefaultStoppingCriteriaFactory.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System; -using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Jobs; - -namespace BenchmarkDotNet.Engines -{ - public class DefaultStoppingCriteriaFactory - { - public static readonly DefaultStoppingCriteriaFactory Instance = new DefaultStoppingCriteriaFactory(); - - private const int MinOverheadIterationCount = 4; - internal const int MaxOverheadIterationCount = 10; - - public virtual IStoppingCriteria CreateWarmupWorkload(Job job, IResolver resolver, RunStrategy runStrategy) - { - var count = job.ResolveValueAsNullable(RunMode.WarmupCountCharacteristic); - if (count.HasValue && count.Value != EngineResolver.ForceAutoWarmup || runStrategy == RunStrategy.Monitoring) - return new FixedStoppingCriteria(count ?? 0); - - int minIterationCount = job.ResolveValue(RunMode.MinWarmupIterationCountCharacteristic, resolver); - int maxIterationCount = job.ResolveValue(RunMode.MaxWarmupIterationCountCharacteristic, resolver); - return new AutoWarmupStoppingCriteria(minIterationCount, maxIterationCount); - } - - public virtual IStoppingCriteria CreateWarmupOverhead() - { - return new AutoWarmupStoppingCriteria(MinOverheadIterationCount, MaxOverheadIterationCount); - } - - public virtual IStoppingCriteria CreateWarmup(Job job, IResolver resolver, IterationMode mode, RunStrategy runStrategy) - { - switch (mode) - { - case IterationMode.Overhead: - return CreateWarmupOverhead(); - case IterationMode.Workload: - return CreateWarmupWorkload(job, resolver, runStrategy); - default: - throw new ArgumentOutOfRangeException(nameof(mode), mode, null); - } - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/FixedStoppingCriteria.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/FixedStoppingCriteria.cs deleted file mode 100644 index 847bb41a59..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/FixedStoppingCriteria.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System.Collections.Generic; -using BenchmarkDotNet.Reports; - -namespace BenchmarkDotNet.Engines -{ - /// - /// Stopping criteria which require a specific amount of iterations. - /// - public class FixedStoppingCriteria : StoppingCriteriaBase - { - private readonly int iterationCount; - - private readonly string message; - - public FixedStoppingCriteria(int iterationCount) - { - this.iterationCount = iterationCount; - - message = $"The required amount of iteration ({iterationCount}) is achieved"; - } - - public override StoppingResult Evaluate(IReadOnlyList measurements) - => measurements.Count >= MaxIterationCount - ? StoppingResult.CreateFinished(message) - : StoppingResult.NotFinished; - - protected override string GetTitle() => $"{nameof(FixedStoppingCriteria)}({nameof(iterationCount)}={iterationCount})"; - - protected override int GetMaxIterationCount() => iterationCount; - - protected override IEnumerable GetWarnings() - { - if (iterationCount < 0) - yield return $"Iteration count ({iterationCount}) is negative"; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/IStoppingCriteria.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/IStoppingCriteria.cs deleted file mode 100644 index 53550ef919..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/IStoppingCriteria.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System.Collections.Generic; -using BenchmarkDotNet.Reports; - -namespace BenchmarkDotNet.Engines -{ - /// - /// A stopping criteria checks when it's time to terminate iteration in the current stage. - /// - public interface IStoppingCriteria - { - /// - /// Checks do we have enough iterations - /// - StoppingResult Evaluate(IReadOnlyList measurements); - - /// - /// Title which can be used in logs and diagnostics methods - /// - string Title { get; } - - /// - /// The maximum possible count of iterations. - /// Engine needs this value for setting the maximum capacity of the returned list of measurements. - /// The correct capacity helps to avoid infrastructure allocations during benchmarking. - /// - int MaxIterationCount { get; } - - /// - /// An array of user-friendly warnings which notify about incorrect parameters. - /// - IReadOnlyList Warnings { get; } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingCriteriaBase.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingCriteriaBase.cs deleted file mode 100644 index 5956e37760..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingCriteriaBase.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using BenchmarkDotNet.Reports; - -namespace BenchmarkDotNet.Engines -{ - public abstract class StoppingCriteriaBase : IStoppingCriteria - { - private readonly Lazy lazyTitle; - private readonly Lazy lazyMaxIterationCount; - private readonly Lazy lazyWarnings; - - public string Title => lazyTitle.Value; - public int MaxIterationCount => lazyMaxIterationCount.Value; - public IReadOnlyList Warnings => lazyWarnings.Value; - - protected StoppingCriteriaBase() - { - lazyTitle = new Lazy(GetTitle); - lazyMaxIterationCount = new Lazy(GetMaxIterationCount); - lazyWarnings = new Lazy(() => GetWarnings().ToArray()); - } - - public abstract StoppingResult Evaluate(IReadOnlyList measurements); - - protected abstract string GetTitle(); - - protected abstract int GetMaxIterationCount(); - - protected abstract IEnumerable GetWarnings(); - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingResult.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingResult.cs deleted file mode 100644 index a51287c4b4..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingResult.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace BenchmarkDotNet.Engines -{ - public struct StoppingResult - { - public readonly bool IsFinished; - - public readonly string? Message; - - private StoppingResult(bool isFinished, string? message) - { - IsFinished = isFinished; - Message = message; - } - - public static readonly StoppingResult NotFinished = new StoppingResult(false, null); - public static StoppingResult CreateFinished(string message) => new StoppingResult(true, message); - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs b/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs index e13d50b650..f2e2f34e26 100644 --- a/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs +++ b/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs @@ -2,10 +2,10 @@ using System.Diagnostics; using System.Linq; using System.Runtime; +using BenchmarkDotNet.Detectors.Cpu; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Portability.Cpu; using BenchmarkDotNet.Properties; using BenchmarkDotNet.Validators; using JetBrains.Annotations; diff --git a/src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs b/src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs index 3b0da64fa3..8291354090 100644 --- a/src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs +++ b/src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs @@ -2,15 +2,19 @@ using System.Collections.Generic; using System.Linq; using System.Text; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Models; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Portability.Cpu; using BenchmarkDotNet.Properties; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Toolchains.DotNetCli; using JetBrains.Annotations; +using Perfolizer.Helpers; using Perfolizer.Horology; +using Perfolizer.Models; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Environments { @@ -25,21 +29,18 @@ public class HostEnvironmentInfo : BenchmarkEnvironmentInfo /// /// Logger to use when there's no config available. /// - public static ILogger FallbackLogger { get; } = ConsoleLogger.Default; + public static ILogger FallbackLogger => ConsoleLogger.Default; - private static HostEnvironmentInfo current; + private static HostEnvironmentInfo? current; public string BenchmarkDotNetVersion { get; protected set; } - /// - /// Could be expensive - /// - public Lazy OsVersion { get; protected set; } - /// /// is expensive to call (1s) /// - public Lazy CpuInfo { get; protected set; } + public Lazy Cpu { get; protected set; } + + public Lazy Os { get; protected set; } /// /// .NET Core SDK version @@ -57,48 +58,54 @@ public class HostEnvironmentInfo : BenchmarkEnvironmentInfo /// The frequency of the timer as the number of ticks per second. /// [PublicAPI] public Frequency ChronometerFrequency { get; protected set; } + [PublicAPI] public TimeInterval ChronometerResolution => ChronometerFrequency.ToResolution(); public HardwareTimerKind HardwareTimerKind { get; protected set; } public Lazy> AntivirusProducts { get; } - public Lazy VirtualMachineHypervisor { get; protected set; } + // TODO: Join with OsInfo + public Lazy VirtualMachineHypervisor { get; protected set; } protected HostEnvironmentInfo() { BenchmarkDotNetVersion = BenchmarkDotNetInfo.Instance.BrandVersion; - OsVersion = new Lazy(RuntimeInformation.GetOsVersion); - CpuInfo = new Lazy(RuntimeInformation.GetCpuInfo); ChronometerFrequency = Chronometer.Frequency; HardwareTimerKind = Chronometer.HardwareTimerKind; DotNetSdkVersion = new Lazy(DotNetCliCommandExecutor.GetDotNetSdkVersion); IsMonoInstalled = new Lazy(() => !string.IsNullOrEmpty(ProcessHelper.RunAndReadOutput("mono", "--version"))); AntivirusProducts = new Lazy>(RuntimeInformation.GetAntivirusProducts); VirtualMachineHypervisor = new Lazy(RuntimeInformation.GetVirtualMachineHypervisor); + Os = new Lazy(OsDetector.GetOs); + Cpu = new Lazy(() => CpuDetector.CrossPlatform.Detect() ?? CpuInfo.Unknown); } - public new static HostEnvironmentInfo GetCurrent() => current ?? (current = new HostEnvironmentInfo()); + public new static HostEnvironmentInfo GetCurrent() => current ??= new HostEnvironmentInfo(); public override IEnumerable ToFormattedString() { - string vmName = VirtualMachineHypervisor.Value?.Name; + string? vmName = VirtualMachineHypervisor.Value?.Name; if (!string.IsNullOrEmpty(vmName)) - yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {OsVersion.Value} ({vmName})"; + yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {Os.Value.ToBrandString()} ({vmName})"; else if (RuntimeInformation.IsRunningInContainer) - yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {OsVersion.Value} (container)"; + yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {Os.Value.ToBrandString()} (container)"; else - yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {OsVersion.Value}"; + yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {Os.Value.ToBrandString()}"; - yield return CpuInfoFormatter.Format(CpuInfo.Value); - var cultureInfo = DefaultCultureInfo.Instance; + yield return Cpu.Value.ToFullBrandName(); if (HardwareTimerKind != HardwareTimerKind.Unknown) - yield return $"Frequency: {ChronometerFrequency}, Resolution: {ChronometerResolution.ToString(cultureInfo)}, Timer: {HardwareTimerKind.ToString().ToUpper()}"; + { + string frequency = ChronometerFrequency.ToString(FrequencyUnit.Hz, unitPresentation: UnitHelper.DefaultPresentation); + string resolution = ChronometerResolution.ToString(UnitHelper.DefaultPresentation); + string timer = HardwareTimerKind.ToString().ToUpper(); + yield return $"Frequency: {frequency}, Resolution: {resolution}, Timer: {timer}"; + } if (RuntimeInformation.IsNetCore && IsDotNetCliInstalled()) { - // this wonderful version number contains words like "preview" and ... 5 segments so it can not be parsed by Version.Parse. Example: "5.0.100-preview.8.20362.3" + // this wonderful version number contains words like "preview" and ... 5 segments, so it can not be parsed by Version.Parse. Example: "5.0.100-preview.8.20362.3" if (int.TryParse(new string(DotNetSdkVersion.Value.TrimStart().TakeWhile(char.IsDigit).ToArray()), out int major) && major >= 5) yield return $".NET SDK {DotNetSdkVersion.Value}"; else @@ -125,5 +132,18 @@ public static string GetInformation() sb.AppendLine(Summary.BuildAllRuntimes(hostEnvironmentInfo, Array.Empty())); return sb.ToString(); } + + internal BdnHostInfo ToPerfonar() => new() + { + Cpu = Cpu.Value, + Os = Os.Value, + RuntimeVersion = RuntimeVersion, + HasAttachedDebugger = HasAttachedDebugger, + HasRyuJit = HasRyuJit, + Configuration = Configuration, + DotNetSdkVersion = DotNetSdkVersion.Value, + ChronometerFrequency = ChronometerFrequency.Hertz, + HardwareTimerKind = HardwareTimerKind.ToString() + }; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs b/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs index 865c6f64f3..dfeaf3fd47 100644 --- a/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs +++ b/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs @@ -17,7 +17,10 @@ private InfrastructureResolver() Register(InfrastructureMode.BuildConfigurationCharacteristic, () => InfrastructureMode.ReleaseConfigurationName); Register(InfrastructureMode.ArgumentsCharacteristic, Array.Empty); + +#pragma warning disable CS0618 // Type or member is obsolete Register(InfrastructureMode.NuGetReferencesCharacteristic, Array.Empty); +#pragma warning restore CS0618 // Type or member is obsolete } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/OsBrandStringHelper.cs b/src/BenchmarkDotNet/Environments/OsBrandStringHelper.cs deleted file mode 100644 index d4369120eb..0000000000 --- a/src/BenchmarkDotNet/Environments/OsBrandStringHelper.cs +++ /dev/null @@ -1,277 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using BenchmarkDotNet.Extensions; - -namespace BenchmarkDotNet.Environments -{ - [SuppressMessage("ReSharper", "ClassNeverInstantiated.Global")] - public class OsBrandStringHelper - { - // See https://en.wikipedia.org/wiki/Ver_(command) - // See https://docs.microsoft.com/en-us/windows/release-health/release-information - // See https://docs.microsoft.com/en-us/windows/release-health/windows11-release-information - private static readonly Dictionary WindowsBrandVersions = new Dictionary - { - { "1.04", "1.0" }, - { "2.11", "2.0" }, - { "3", "3.0" }, - { "3.10.528", "NT 3.1" }, - { "3.11", "for Workgroups 3.11" }, - { "3.50.807", "NT 3.5" }, - { "3.51.1057", "NT 3.51" }, - { "4.00.950", "95" }, - { "4.00.1111", "95 OSR2" }, - { "4.03.1212-1214", "95 OSR2.1" }, - { "4.03.1214", "95 OSR2.5" }, - { "4.00.1381", "NT 4.0" }, - { "4.10.1998", "98" }, - { "4.10.2222", "98 SE" }, - { "4.90.2380.2", "ME Beta" }, - { "4.90.2419", "ME Beta 2" }, - { "4.90.3000", "ME" }, - { "5.00.1515", "NT 5.0 Beta" }, - { "5.00.2031", "2000 Beta 3" }, - { "5.00.2128", "2000 RC2" }, - { "5.00.2183", "2000 RC3" }, - { "5.00.2195", "2000" }, - { "5.0.2195", "2000 Professional" }, - { "5.1.2505", "XP RC1" }, - { "5.1.2600", "XP" }, - { "5.1.2600.1105-1106", "XP SP1" }, - { "5.1.2600.2180", "XP SP2" }, - { "5.2.3541", ".NET Server interim" }, - { "5.2.3590", ".NET Server Beta 3" }, - { "5.2.3660", ".NET Server RC1" }, - { "5.2.3718", ".NET Server 2003 RC2" }, - { "5.2.3763", "Server 2003 Beta" }, - { "5.2.3790", "XP Professional x64 Edition" }, - { "5.2.3790.1180", "Server 2003 SP1" }, - { "5.2.3790.1218", "Server 2003" }, - { "6.0.5048", "Longhorn" }, - { "6.0.5112", "Vista Beta 1" }, - { "6.0.5219", "Vista CTP" }, - { "6.0.5259", "Vista TAP Preview" }, - { "6.0.5270", "Vista CTP December" }, - { "6.0.5308", "Vista CTP February" }, - { "6.0.5342", "Vista CTP Refresh" }, - { "6.0.5365", "Vista April EWD" }, - { "6.0.5381", "Vista Beta 2 Preview" }, - { "6.0.5384", "Vista Beta 2" }, - { "6.0.5456", "Vista Pre-RC1 Build 5456" }, - { "6.0.5472", "Vista Pre-RC1 Build 5472" }, - { "6.0.5536", "Vista Pre-RC1 Build 5536" }, - { "6.0.5600.16384", "Vista RC1" }, - { "6.0.5700", "Vista Pre-RC2" }, - { "6.0.5728", "Vista Pre-RC2 Build 5728" }, - { "6.0.5744.16384", "Vista RC2" }, - { "6.0.5808", "Vista Pre-RTM Build 5808" }, - { "6.0.5824", "Vista Pre-RTM Build 5824" }, - { "6.0.5840", "Vista Pre-RTM Build 5840" }, - { "6.0.6000", "Vista" }, - { "6.0.6000.16386", "Vista RTM" }, - { "6.0.6001", "Vista SP1" }, - { "6.0.6002", "Vista SP2" }, - { "6.1.7600", "7" }, - { "6.1.7600.16385", "7" }, - { "6.1.7601", "7 SP1" }, - { "6.1.8400", "Home Server 2011" }, - { "6.2.8102", "8 Developer Preview" }, - { "6.2.9200", "8" }, - { "6.2.9200.16384", "8 RTM" }, - { "6.2.10211", "Phone 8" }, - { "6.3.9600", "8.1" }, - { "6.4.9841", "10 Technical Preview 1" }, - { "6.4.9860", "10 Technical Preview 2" }, - { "6.4.9879", "10 Technical Preview 3" }, - { "10.0.9926", "10 Technical Preview 4" }, - { "10.0.10041", "10 Technical Preview 5" }, - { "10.0.10049", "10 Technical Preview 6" }, - { "10.0.10240", "10 Threshold 1 [1507, RTM]" }, - { "10.0.10586", "10 Threshold 2 [1511, November Update]" }, - { "10.0.14393", "10 Redstone 1 [1607, Anniversary Update]" }, - { "10.0.15063", "10 Redstone 2 [1703, Creators Update]" }, - { "10.0.16299", "10 Redstone 3 [1709, Fall Creators Update]" }, - { "10.0.17134", "10 Redstone 4 [1803, April 2018 Update]" }, - { "10.0.17763", "10 Redstone 5 [1809, October 2018 Update]" }, - { "10.0.18362", "10 19H1 [1903, May 2019 Update]" }, - { "10.0.18363", "10 19H2 [1909, November 2019 Update]" }, - { "10.0.19041", "10 20H1 [2004, May 2020 Update]" }, - { "10.0.19042", "10 20H2 [20H2, October 2020 Update]" }, - { "10.0.19043", "10 21H1 [21H1, May 2021 Update]" }, - { "10.0.19044", "10 21H2 [21H2, November 2021 Update]" }, - { "10.0.19045", "10 22H2 [22H2, 2022 Update]" }, - { "10.0.22000", "11 21H2 [21H2, Sun Valley]" }, - { "10.0.22621", "11 22H2 [22H2, Sun Valley 2]" }, - }; - - private class Windows1XVersion - { - private string? CodeVersion { get; } - private string? CodeName { get; } - private string? MarketingName { get; } - private int BuildNumber { get; } - - private string MarketingNumber => BuildNumber >= 22000 ? "11" : "10"; - private string? ShortifiedCodeName => CodeName?.Replace(" ", ""); - private string? ShortifiedMarketingName => MarketingName?.Replace(" ", ""); - - private Windows1XVersion(string? codeVersion, string? codeName, string? marketingName, int buildNumber) - { - CodeVersion = codeVersion; - CodeName = codeName; - MarketingName = marketingName; - BuildNumber = buildNumber; - } - - private string ToFullVersion(int? ubr = null) - => ubr == null ? $"10.0.{BuildNumber}" : $"10.0.{BuildNumber}.{ubr}"; - - private static string Collapse(params string[] values) => string.Join("/", values.Where(v => !string.IsNullOrEmpty(v))); - - // The line with OsBrandString is one of the longest lines in the summary. - // When people past in on GitHub, it can be a reason of an ugly horizontal scrollbar. - // To avoid this, we are trying to minimize this line and use the minimum possible number of characters. - public string ToPrettifiedString(int? ubr) - => CodeVersion == ShortifiedCodeName - ? $"{MarketingNumber} ({Collapse(ToFullVersion(ubr), CodeVersion, ShortifiedMarketingName)})" - : $"{MarketingNumber} ({Collapse(ToFullVersion(ubr), CodeVersion, ShortifiedMarketingName, ShortifiedCodeName)})"; - - // See https://en.wikipedia.org/wiki/Windows_10_version_history - // See https://en.wikipedia.org/wiki/Windows_11_version_history - private static readonly List WellKnownVersions = new () - { - // Windows 10 - new Windows1XVersion("1507", "Threshold 1", "RTM", 10240), - new Windows1XVersion("1511", "Threshold 2", "November Update", 10586), - new Windows1XVersion("1607", "Redstone 1", "Anniversary Update", 14393), - new Windows1XVersion("1703", "Redstone 2", "Creators Update", 15063), - new Windows1XVersion("1709", "Redstone 3", "Fall Creators Update", 16299), - new Windows1XVersion("1803", "Redstone 4", "April 2018 Update", 17134), - new Windows1XVersion("1809", "Redstone 5", "October 2018 Update", 17763), - new Windows1XVersion("1903", "19H1", "May 2019 Update", 18362), - new Windows1XVersion("1909", "19H2", "November 2019 Update", 18363), - new Windows1XVersion("2004", "20H1", "May 2020 Update", 19041), - new Windows1XVersion("20H2", "20H2", "October 2020 Update", 19042), - new Windows1XVersion("21H1", "21H1", "May 2021 Update", 19043), - new Windows1XVersion("21H2", "21H2", "November 2021 Update", 19044), - new Windows1XVersion("22H2", "22H2", "2022 Update", 19045), - // Windows 11 - new Windows1XVersion("21H2", "Sun Valley", null, 22000), - new Windows1XVersion("22H2", "Sun Valley 2", "2022 Update", 22621), - new Windows1XVersion("23H2", "Sun Valley 3", "2023 Update", 22631), - }; - - public static Windows1XVersion? Resolve(string osVersionString) - { - var windows1XVersion = WellKnownVersions.FirstOrDefault(v => osVersionString == $"10.0.{v.BuildNumber}"); - if (windows1XVersion != null) - return windows1XVersion; - if (Version.TryParse(osVersionString, out var osVersion)) - { - if (osVersion.Major == 10 && osVersion.Minor == 0) - return new Windows1XVersion(null, null, null, osVersion.Build); - } - return null; - } - } - - /// - /// Transform an operation system name and version to a nice form for summary. - /// - /// Original operation system name - /// Original operation system version - /// UBR (Update Build Revision), the revision number of Windows version (if available) - /// Prettified operation system title - public static string Prettify(string osName, string osVersion, int? windowsUbr = null) - { - if (osName == "Windows") - return PrettifyWindows(osVersion, windowsUbr); - return $"{osName} {osVersion}"; - } - - private static string PrettifyWindows(string osVersion, int? windowsUbr) - { - var windows1XVersion = Windows1XVersion.Resolve(osVersion); - if (windows1XVersion != null) - return "Windows " + windows1XVersion.ToPrettifiedString(windowsUbr); - - string brandVersion = WindowsBrandVersions.GetValueOrDefault(osVersion); - string completeOsVersion = windowsUbr != null && osVersion.Count(c => c == '.') == 2 - ? osVersion + "." + windowsUbr - : osVersion; - string fullVersion = brandVersion == null ? osVersion : brandVersion + " (" + completeOsVersion + ")"; - return "Windows " + fullVersion; - } - - private class MacOSXVersion - { - private int DarwinVersion { get; } - private string CodeName { get; } - - private MacOSXVersion(int darwinVersion, string codeName) - { - DarwinVersion = darwinVersion; - CodeName = codeName; - } - - private static readonly List WellKnownVersions = new List - { - new MacOSXVersion(6, "Jaguar"), - new MacOSXVersion(7, "Panther"), - new MacOSXVersion(8, "Tiger"), - new MacOSXVersion(9, "Leopard"), - new MacOSXVersion(10, "Snow Leopard"), - new MacOSXVersion(11, "Lion"), - new MacOSXVersion(12, "Mountain Lion"), - new MacOSXVersion(13, "Mavericks"), - new MacOSXVersion(14, "Yosemite"), - new MacOSXVersion(15, "El Capitan"), - new MacOSXVersion(16, "Sierra"), - new MacOSXVersion(17, "High Sierra"), - new MacOSXVersion(18, "Mojave"), - new MacOSXVersion(19, "Catalina"), - new MacOSXVersion(20, "Big Sur"), - new MacOSXVersion(21, "Monterey"), - new MacOSXVersion(22, "Ventura"), - new MacOSXVersion(23, "Sonoma"), - }; - - public static string? ResolveCodeName(string kernelVersion) - { - if (string.IsNullOrWhiteSpace(kernelVersion)) - return null; - - kernelVersion = kernelVersion.ToLowerInvariant().Trim(); - if (kernelVersion.StartsWith("darwin")) - kernelVersion = kernelVersion.Substring(6).Trim(); - var numbers = kernelVersion.Split('.'); - if (numbers.Length == 0) - return null; - - string majorVersionStr = numbers[0]; - if (int.TryParse(majorVersionStr, out int majorVersion)) - return WellKnownVersions.FirstOrDefault(v => v.DarwinVersion == majorVersion)?.CodeName; - return null; - } - } - - public static string PrettifyMacOSX(string systemVersion, string kernelVersion) - { - string codeName = MacOSXVersion.ResolveCodeName(kernelVersion); - if (codeName != null) - { - int firstDigitIndex = systemVersion.IndexOfAny("0123456789".ToCharArray()); - if (firstDigitIndex == -1) - return $"{systemVersion} {codeName} [{kernelVersion}]"; - - string systemVersionTitle = systemVersion.Substring(0, firstDigitIndex).Trim(); - string systemVersionNumbers = systemVersion.Substring(firstDigitIndex).Trim(); - return $"{systemVersionTitle} {codeName} {systemVersionNumbers} [{kernelVersion}]"; - } - - return $"{systemVersion} [{kernelVersion}]"; - } - } -} diff --git a/src/BenchmarkDotNet/Environments/Platform.cs b/src/BenchmarkDotNet/Environments/Platform.cs index bc8f1ad26b..434a84ff98 100644 --- a/src/BenchmarkDotNet/Environments/Platform.cs +++ b/src/BenchmarkDotNet/Environments/Platform.cs @@ -51,5 +51,10 @@ public enum Platform /// A PowerPC 64-bit (little-endian) processor architecture. /// Ppc64le, + + /// + /// A RiscV 64-bit processor architecture. + /// + RiscV64, } } diff --git a/src/BenchmarkDotNet/Environments/ProcessorBrandStringHelper.cs b/src/BenchmarkDotNet/Environments/ProcessorBrandStringHelper.cs deleted file mode 100644 index 882078df74..0000000000 --- a/src/BenchmarkDotNet/Environments/ProcessorBrandStringHelper.cs +++ /dev/null @@ -1,155 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Text.RegularExpressions; -using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Portability.Cpu; -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Environments -{ - public static class ProcessorBrandStringHelper - { - /// - /// Transform a processor brand string to a nice form for summary. - /// - /// The CPU information - /// Whether to include determined max frequency information - /// Prettified version - public static string Prettify(CpuInfo cpuInfo, bool includeMaxFrequency = false) - { - if (cpuInfo == null || string.IsNullOrEmpty(cpuInfo.ProcessorName)) - { - return "Unknown processor"; - } - - // Remove parts which don't provide any useful information for user - var processorName = cpuInfo.ProcessorName.Replace("@", "").Replace("(R)", "").Replace("(TM)", ""); - - // If we have found physical core(s), we can safely assume we can drop extra info from brand - if (cpuInfo.PhysicalCoreCount.HasValue && cpuInfo.PhysicalCoreCount.Value > 0) - processorName = Regex.Replace(processorName, @"(\w+?-Core Processor)", "").Trim(); - - string frequencyString = GetBrandStyledActualFrequency(cpuInfo.NominalFrequency); - if (includeMaxFrequency && frequencyString != null && !processorName.Contains(frequencyString)) - { - // show Max only if there's already a frequency name to differentiate the two - string maxFrequency = processorName.Contains("Hz") ? $"(Max: {frequencyString})" : frequencyString; - processorName = $"{processorName} {maxFrequency}"; - } - - // Remove double spaces - processorName = Regex.Replace(processorName.Trim(), @"\s+", " "); - - // Add microarchitecture name if known - string microarchitecture = ParseMicroarchitecture(processorName); - if (microarchitecture != null) - processorName = $"{processorName} ({microarchitecture})"; - - return processorName; - } - - /// - /// Presents actual processor's frequency into brand string format - /// - /// - private static string GetBrandStyledActualFrequency(Frequency? frequency) - { - if (frequency == null) - return null; - return $"{frequency.Value.ToGHz().ToString("N2", DefaultCultureInfo.Instance)}GHz"; - } - - /// - /// Parse a processor name and tries to return a microarchitecture name. - /// Works only for well-known microarchitectures. - /// - private static string? ParseMicroarchitecture(string processorName) - { - if (processorName.StartsWith("Intel Core")) - { - string model = processorName.Substring("Intel Core".Length).Trim(); - - // Core i3/5/7/9 - if ( - model.Length > 4 && - model[0] == 'i' && - (model[1] == '3' || model[1] == '5' || model[1] == '7' || model[1] == '9') && - (model[2] == '-' || model[2] == ' ')) - { - string modelNumber = model.Substring(3); - if (modelNumber.StartsWith("CPU")) - modelNumber = modelNumber.Substring(3).Trim(); - if (modelNumber.Contains("CPU")) - modelNumber = modelNumber.Substring(0, modelNumber.IndexOf("CPU", StringComparison.Ordinal)).Trim(); - return ParseIntelCoreMicroarchitecture(modelNumber); - } - } - - return null; - } - - private static readonly Lazy> KnownMicroarchitectures = new Lazy>(() => - { - var data = ResourceHelper.LoadResource("BenchmarkDotNet.Environments.microarchitectures.txt").Split('\r', '\n'); - var dictionary = new Dictionary(); - string? currentMicroarchitecture = null; - foreach (string line in data) - { - if (line.StartsWith("//") || string.IsNullOrWhiteSpace(line)) - continue; - if (line.StartsWith("#")) - { - currentMicroarchitecture = line.Substring(1).Trim(); - continue; - } - - string modelNumber = line.Trim(); - if (dictionary.ContainsKey(modelNumber)) - throw new Exception($"{modelNumber} is defined twice in microarchitectures.txt"); - if (currentMicroarchitecture == null) - throw new Exception($"{modelNumber} doesn't have defined microarchitecture in microarchitectures.txt"); - dictionary[modelNumber] = currentMicroarchitecture; - } - - return dictionary; - }); - - // see http://www.intel.com/content/www/us/en/processors/processor-numbers.html - [SuppressMessage("ReSharper", "StringLiteralTypo")] - internal static string? ParseIntelCoreMicroarchitecture(string modelNumber) - { - if (KnownMicroarchitectures.Value.TryGetValue(modelNumber, out string? microarchitecture)) - return microarchitecture; - - if (modelNumber.Length >= 3 && modelNumber.Substring(0, 3).All(char.IsDigit) && - (modelNumber.Length == 3 || !char.IsDigit(modelNumber[3]))) - return "Nehalem"; - if (modelNumber.Length >= 4 && modelNumber.Substring(0, 4).All(char.IsDigit)) - { - char generation = modelNumber[0]; - switch (generation) - { - case '2': - return "Sandy Bridge"; - case '3': - return "Ivy Bridge"; - case '4': - return "Haswell"; - case '5': - return "Broadwell"; - case '6': - return "Skylake"; - case '7': - return "Kaby Lake"; - case '8': - return "Coffee Lake"; - default: - return null; - } - } - return null; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs index be5a84cce2..21bb2821df 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs @@ -1,4 +1,5 @@ using System; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -44,7 +45,7 @@ public static ClrRuntime CreateForLocalFullNetFrameworkBuild(string version) internal static ClrRuntime GetCurrentVersion() { - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) { throw new NotSupportedException(".NET Framework supports Windows OS only."); } diff --git a/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs index d86e4b0f40..472b629405 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs @@ -11,18 +11,19 @@ namespace BenchmarkDotNet.Environments { public class CoreRuntime : Runtime { - public static readonly CoreRuntime Core20 = new (RuntimeMoniker.NetCoreApp20, "netcoreapp2.0", ".NET Core 2.0"); - public static readonly CoreRuntime Core21 = new (RuntimeMoniker.NetCoreApp21, "netcoreapp2.1", ".NET Core 2.1"); - public static readonly CoreRuntime Core22 = new (RuntimeMoniker.NetCoreApp22, "netcoreapp2.2", ".NET Core 2.2"); - public static readonly CoreRuntime Core30 = new (RuntimeMoniker.NetCoreApp30, "netcoreapp3.0", ".NET Core 3.0"); - public static readonly CoreRuntime Core31 = new (RuntimeMoniker.NetCoreApp31, "netcoreapp3.1", ".NET Core 3.1"); - public static readonly CoreRuntime Core50 = new (RuntimeMoniker.Net50, "net5.0", ".NET 5.0"); - public static readonly CoreRuntime Core60 = new (RuntimeMoniker.Net60, "net6.0", ".NET 6.0"); - public static readonly CoreRuntime Core70 = new (RuntimeMoniker.Net70, "net7.0", ".NET 7.0"); - public static readonly CoreRuntime Core80 = new (RuntimeMoniker.Net80, "net8.0", ".NET 8.0"); - public static readonly CoreRuntime Core90 = new (RuntimeMoniker.Net90, "net9.0", ".NET 9.0"); - - public static CoreRuntime Latest => Core90; // when dotnet/runtime branches for 10.0, this will need to get updated + public static readonly CoreRuntime Core20 = new(RuntimeMoniker.NetCoreApp20, "netcoreapp2.0", ".NET Core 2.0"); + public static readonly CoreRuntime Core21 = new(RuntimeMoniker.NetCoreApp21, "netcoreapp2.1", ".NET Core 2.1"); + public static readonly CoreRuntime Core22 = new(RuntimeMoniker.NetCoreApp22, "netcoreapp2.2", ".NET Core 2.2"); + public static readonly CoreRuntime Core30 = new(RuntimeMoniker.NetCoreApp30, "netcoreapp3.0", ".NET Core 3.0"); + public static readonly CoreRuntime Core31 = new(RuntimeMoniker.NetCoreApp31, "netcoreapp3.1", ".NET Core 3.1"); + public static readonly CoreRuntime Core50 = new(RuntimeMoniker.Net50, "net5.0", ".NET 5.0"); + public static readonly CoreRuntime Core60 = new(RuntimeMoniker.Net60, "net6.0", ".NET 6.0"); + public static readonly CoreRuntime Core70 = new(RuntimeMoniker.Net70, "net7.0", ".NET 7.0"); + public static readonly CoreRuntime Core80 = new(RuntimeMoniker.Net80, "net8.0", ".NET 8.0"); + public static readonly CoreRuntime Core90 = new(RuntimeMoniker.Net90, "net9.0", ".NET 9.0"); + public static readonly CoreRuntime Core10_0 = new(RuntimeMoniker.Net10_0, "net10.0", ".NET 10.0"); + + public static CoreRuntime Latest => Core10_0; // when dotnet/runtime branches for 11.0, this will need to get updated private CoreRuntime(RuntimeMoniker runtimeMoniker, string msBuildMoniker, string displayName) : base(runtimeMoniker, msBuildMoniker, displayName) @@ -74,6 +75,7 @@ internal static CoreRuntime FromVersion(Version version) case Version v when v.Major == 7 && v.Minor == 0: return GetPlatformSpecific(Core70); case Version v when v.Major == 8 && v.Minor == 0: return GetPlatformSpecific(Core80); case Version v when v.Major == 9 && v.Minor == 0: return GetPlatformSpecific(Core90); + case Version v when v.Major == 10 && v.Minor == 0: return GetPlatformSpecific(Core10_0); default: return CreateForNewVersion($"net{version.Major}.{version.Minor}", $".NET {version.Major}.{version.Minor}"); } @@ -97,11 +99,26 @@ internal static bool TryGetVersion(out Version? version) return true; } - var systemPrivateCoreLib = FileVersionInfo.GetVersionInfo(typeof(object).Assembly.Location); - // systemPrivateCoreLib.Product*Part properties return 0 so we have to implement some ugly parsing... - if (TryGetVersionFromProductInfo(systemPrivateCoreLib.ProductVersion, systemPrivateCoreLib.ProductName, out version)) + string coreclrLocation = typeof(object).Assembly.Location; + // Single-file publish has empty assembly location. + if (!string.IsNullOrEmpty(coreclrLocation)) { - return true; + var systemPrivateCoreLib = FileVersionInfo.GetVersionInfo(coreclrLocation); + // systemPrivateCoreLib.Product*Part properties return 0 so we have to implement some ugly parsing... + if (TryGetVersionFromProductInfo(systemPrivateCoreLib.ProductVersion, systemPrivateCoreLib.ProductName, out version)) + { + return true; + } + } + else + { + // .Net Core 3.X supports single-file publish, .Net Core 2.X does not. + // .Net Core 3.X fixed the version in FrameworkDescription, so we don't need to handle the case of 4.6.x in this branch. + var frameworkDescriptionVersion = GetParsableVersionPart(GetVersionFromFrameworkDescription()); + if (Version.TryParse(frameworkDescriptionVersion, out version)) + { + return true; + } } // it's OK to use this method only after checking the previous ones @@ -123,6 +140,14 @@ internal static bool TryGetVersion(out Version? version) return false; } + internal static string GetVersionFromFrameworkDescription() + { + // .NET 10.0.0-preview.5.25277.114 -> 10.0.0-preview.5.25277.114 + // .NET Core 3.1.32 -> 3.1.32 + string frameworkDescription = System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription; + return new string(frameworkDescription.SkipWhile(c => !char.IsDigit(c)).ToArray()); + } + // sample input: // for dotnet run: C:\Program Files\dotnet\shared\Microsoft.NETCore.App\2.1.12\ // for dotnet publish: C:\Users\adsitnik\source\repos\ConsoleApp25\ConsoleApp25\bin\Release\netcoreapp2.0\win-x64\publish\ @@ -192,7 +217,7 @@ internal static bool TryGetVersionFromFrameworkName(string frameworkName, out Ve } // Version.TryParse does not handle thing like 3.0.0-WORD - private static string GetParsableVersionPart(string fullVersionName) => new string(fullVersionName.TakeWhile(c => char.IsDigit(c) || c == '.').ToArray()); + internal static string GetParsableVersionPart(string fullVersionName) => new string(fullVersionName.TakeWhile(c => char.IsDigit(c) || c == '.').ToArray()); private static CoreRuntime GetPlatformSpecific(CoreRuntime fallback) { diff --git a/src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs index e81e64f6c6..0b5d2c159b 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs @@ -20,7 +20,7 @@ public class MonoAotLLVMRuntime : Runtime, IEquatable /// /// creates new instance of MonoAotLLVMRuntime /// - public MonoAotLLVMRuntime(FileInfo aotCompilerPath, MonoAotCompilerMode aotCompilerMode, string msBuildMoniker = "net6.0", string displayName = "MonoAOTLLVM") : base(RuntimeMoniker.MonoAOTLLVM, msBuildMoniker, displayName) + public MonoAotLLVMRuntime(FileInfo aotCompilerPath, MonoAotCompilerMode aotCompilerMode, string msBuildMoniker = "net6.0", string displayName = "MonoAOTLLVM", RuntimeMoniker moniker = RuntimeMoniker.MonoAOTLLVM) : base(moniker, msBuildMoniker, displayName) { if (aotCompilerPath == null) throw new ArgumentNullException(paramName: nameof(aotCompilerPath)); diff --git a/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs index 5862d1390c..c1c37db2cd 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs @@ -5,11 +5,12 @@ namespace BenchmarkDotNet.Environments { public class MonoRuntime : Runtime, IEquatable { - public static readonly MonoRuntime Default = new ("Mono"); - public static readonly MonoRuntime Mono60 = new ("Mono with .NET 6.0", RuntimeMoniker.Mono60, "net6.0", isDotNetBuiltIn: true); - public static readonly MonoRuntime Mono70 = new ("Mono with .NET 7.0", RuntimeMoniker.Mono70, "net7.0", isDotNetBuiltIn: true); - public static readonly MonoRuntime Mono80 = new ("Mono with .NET 8.0", RuntimeMoniker.Mono80, "net8.0", isDotNetBuiltIn: true); - public static readonly MonoRuntime Mono90 = new ("Mono with .NET 9.0", RuntimeMoniker.Mono90, "net9.0", isDotNetBuiltIn: true); + public static readonly MonoRuntime Default = new("Mono"); + public static readonly MonoRuntime Mono60 = new("Mono with .NET 6.0", RuntimeMoniker.Mono60, "net6.0", isDotNetBuiltIn: true); + public static readonly MonoRuntime Mono70 = new("Mono with .NET 7.0", RuntimeMoniker.Mono70, "net7.0", isDotNetBuiltIn: true); + public static readonly MonoRuntime Mono80 = new("Mono with .NET 8.0", RuntimeMoniker.Mono80, "net8.0", isDotNetBuiltIn: true); + public static readonly MonoRuntime Mono90 = new("Mono with .NET 9.0", RuntimeMoniker.Mono90, "net9.0", isDotNetBuiltIn: true); + public static readonly MonoRuntime Mono10_0 = new("Mono with .NET 10.0", RuntimeMoniker.Mono10_0, "net10.0", isDotNetBuiltIn: true); public string CustomPath { get; } diff --git a/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs index f1f49c5dca..bff018558e 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs @@ -22,6 +22,10 @@ public class NativeAotRuntime : Runtime /// NativeAOT compiled as net9.0 /// public static readonly NativeAotRuntime Net90 = new NativeAotRuntime(RuntimeMoniker.NativeAot90, "net9.0", "NativeAOT 9.0"); + /// + /// NativeAOT compiled as net10.0 + /// + public static readonly NativeAotRuntime Net10_0 = new NativeAotRuntime(RuntimeMoniker.NativeAot10_0, "net10.0", "NativeAOT 10.0"); public override bool IsAOT => true; @@ -48,6 +52,7 @@ public static NativeAotRuntime GetCurrentVersion() case Version v when v.Major == 7 && v.Minor == 0: return Net70; case Version v when v.Major == 8 && v.Minor == 0: return Net80; case Version v when v.Major == 9 && v.Minor == 0: return Net90; + case Version v when v.Major == 10 && v.Minor == 0: return Net10_0; default: return new NativeAotRuntime(RuntimeMoniker.NotRecognized, $"net{version.Major}.{version.Minor}", $"NativeAOT {version.Major}.{version.Minor}"); } diff --git a/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs b/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs index b695de321c..4a46e6456d 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs @@ -18,7 +18,7 @@ public abstract class Runtime : IEquatable public RuntimeMoniker RuntimeMoniker { get; } /// - /// MsBuild Target Framework Moniker, example: net462, netcoreapp2.1 + /// MsBuild Target Framework Moniker, example: net462, net8.0 /// public string MsBuildMoniker { get; } diff --git a/src/BenchmarkDotNet/Environments/microarchitectures.txt b/src/BenchmarkDotNet/Environments/microarchitectures.txt deleted file mode 100644 index f06d295390..0000000000 --- a/src/BenchmarkDotNet/Environments/microarchitectures.txt +++ /dev/null @@ -1,183 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Kaby Lake -// See: https://en.wikipedia.org/wiki/Kaby_Lake -//////////////////////////////////////////////////////////////////////////////// - -# Kaby Lake -7Y30 -7Y32 -7Y54 -7Y57 -7Y75 -3865U -3965U -4410Y -4415U -7100 -7100H -7100T -7100U -7101E -7101TE -7130U -7167U -7200U -7260U -7267U -7287U -7300 -7300HQ -7300T -7300U -7320 -7350K -7360U -7400 -7400T -7440HQ -7500 -7500T -7500U -7560U -7567U -7600 -7600K -7600T -7600U -7640X -7660U -7700 -7700HQ -7700K -7700T -7740X -7820HK -7820HQ -7920HQ -8130U -E3-1220 v6 -E3-1225 v6 -E3-1230 v6 -E3-1240 v6 -E3-1245 v6 -E3-1270 v6 -E3-1275 v6 -E3-1280 v6 -E3-1285 v6 -E3-1505L v6 -E3-1505M v6 -E3-1535M v6 -G3930 -G3930T -G3950 -G4560 -G4560T -G4600 -G4600T -G4620 - -# Kaby Lake R -8250U -8350U -8550U -8650U - -# Kaby Lake G -8305G -8705G -8706G -8709G -8809G - -# Amber Lake Y -8100Y -8200Y -8210Y -8500Y - -//////////////////////////////////////////////////////////////////////////////// -// Coffee Lake -// See: https://en.wikipedia.org/wiki/Coffee_Lake -//////////////////////////////////////////////////////////////////////////////// - -# Coffee Lake -610 -2104G -2124 -2124G -2126G -2134 -2136 -2144G -2146G -2174G -2176G -2176M -2186G -2186M -8086K -8100 -8100H -8100T -8109U -8259U -8269U -8300 -8300H -8300T -8350K -8400 -8400B -8400H -8400T -8500 -8500B -8500T -8559U -8600 -8600K -8600T -8700 -8700B -8700K -8700T -8750H -8850H -8950HK -9350KF -9400 -9400F -9600K -9600KF -9700K -9700KF -9900K -9900KF -G4900 -G4900T -G4920 -G5400 -G5400T -G5500 -G5500T -G5600 - -//////////////////////////////////////////////////////////////////////////////// -// Cannon Lake -// See: https://en.wikipedia.org/wiki/Cannon_Lake_(microarchitecture) -//////////////////////////////////////////////////////////////////////////////// - -# Cannon Lake -8121U - -//////////////////////////////////////////////////////////////////////////////// -// Whiskey Lake -// See: https://en.wikipedia.org/wiki/Whiskey_Lake_(microarchitecture) -//////////////////////////////////////////////////////////////////////////////// - -# Whiskey Lake -8565U -8265U -8145U -5405U -4205U \ No newline at end of file diff --git a/src/BenchmarkDotNet/Exporters/ExporterBase.cs b/src/BenchmarkDotNet/Exporters/ExporterBase.cs index 17a8d432a6..a7604b2864 100644 --- a/src/BenchmarkDotNet/Exporters/ExporterBase.cs +++ b/src/BenchmarkDotNet/Exporters/ExporterBase.cs @@ -48,7 +48,7 @@ public IEnumerable ExportToFiles(Summary summary, ILogger consoleLogger) return new[] { filePath }; } - internal string GetArtifactFullName(Summary summary) + public string GetArtifactFullName(Summary summary) { string fileName = GetFileName(summary); return $"{Path.Combine(summary.ResultsDirectoryPath, fileName)}-{FileCaption}{FileNameSuffix}.{FileExtension}"; diff --git a/src/BenchmarkDotNet/Exporters/Json/JsonExporterBase.cs b/src/BenchmarkDotNet/Exporters/Json/JsonExporterBase.cs index 8ce4472a5a..3727ecac80 100644 --- a/src/BenchmarkDotNet/Exporters/Json/JsonExporterBase.cs +++ b/src/BenchmarkDotNet/Exporters/Json/JsonExporterBase.cs @@ -3,8 +3,9 @@ using BenchmarkDotNet.Environments; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; +using Perfolizer.Helpers; using Perfolizer.Horology; -using JsonSerializer = SimpleJson.SimpleJson; +using SimpleJson; namespace BenchmarkDotNet.Exporters.Json { @@ -23,8 +24,8 @@ protected JsonExporterBase(bool indentJson = false, bool excludeMeasurements = f public override void ExportToLog(Summary summary, ILogger logger) { - JsonSerializer.CurrentJsonSerializerStrategy.Indent = IndentJson; - logger.WriteLine(JsonSerializer.SerializeObject(GetDataToSerialize(summary))); + SimpleJsonSerializer.CurrentJsonSerializerStrategy.Indent = IndentJson; + logger.WriteLine(SimpleJsonSerializer.SerializeObject(GetDataToSerialize(summary))); } protected virtual IReadOnlyDictionary GetDataToSerialize(Summary summary) @@ -47,11 +48,11 @@ protected virtual IReadOnlyDictionary GetDataToSerialize(HostEnv { { nameof(HostEnvironmentInfo.BenchmarkDotNetCaption), HostEnvironmentInfo.BenchmarkDotNetCaption }, { nameof(environmentInfo.BenchmarkDotNetVersion), environmentInfo.BenchmarkDotNetVersion }, - { "OsVersion", environmentInfo.OsVersion.Value }, - { "ProcessorName", ProcessorBrandStringHelper.Prettify(environmentInfo.CpuInfo.Value) }, - { "PhysicalProcessorCount", environmentInfo.CpuInfo.Value?.PhysicalProcessorCount }, - { "PhysicalCoreCount", environmentInfo.CpuInfo.Value?.PhysicalCoreCount }, - { "LogicalCoreCount", environmentInfo.CpuInfo.Value?.LogicalCoreCount }, + { "OsVersion", environmentInfo.Os.Value.ToBrandString() }, + { "ProcessorName", environmentInfo.Cpu.Value.ToShortBrandName() }, + { "PhysicalProcessorCount", environmentInfo.Cpu.Value?.PhysicalProcessorCount }, + { "PhysicalCoreCount", environmentInfo.Cpu.Value?.PhysicalCoreCount }, + { "LogicalCoreCount", environmentInfo.Cpu.Value?.LogicalCoreCount }, { nameof(environmentInfo.RuntimeVersion), environmentInfo.RuntimeVersion }, { nameof(environmentInfo.Architecture), environmentInfo.Architecture }, { nameof(environmentInfo.HasAttachedDebugger), environmentInfo.HasAttachedDebugger }, @@ -83,40 +84,40 @@ protected virtual IReadOnlyDictionary GetDataToSerialize(Benchma { "Statistics", report.ResultStatistics } }; - // We show MemoryDiagnoser's results only if it is being used - if (report.BenchmarkCase.Config.HasMemoryDiagnoser()) + // We show MemoryDiagnoser's results only if it is being used + if (report.BenchmarkCase.Config.HasMemoryDiagnoser()) + { + benchmark.Add("Memory", new { - benchmark.Add("Memory", new + report.GcStats.Gen0Collections, + report.GcStats.Gen1Collections, + report.GcStats.Gen2Collections, + report.GcStats.TotalOperations, + BytesAllocatedPerOperation = report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) + }); + } + + if (ExcludeMeasurements == false) + { + // We construct Measurements manually, so that we can have the IterationMode enum as text, rather than an integer + benchmark.Add("Measurements", + report.AllMeasurements.Select(m => new { - report.GcStats.Gen0Collections, - report.GcStats.Gen1Collections, - report.GcStats.Gen2Collections, - report.GcStats.TotalOperations, - BytesAllocatedPerOperation = report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) - }); - } + IterationMode = m.IterationMode.ToString(), + IterationStage = m.IterationStage.ToString(), + m.LaunchIndex, + m.IterationIndex, + m.Operations, + m.Nanoseconds + })); - if (ExcludeMeasurements == false) + if (report.Metrics.Any()) { - // We construct Measurements manually, so that we can have the IterationMode enum as text, rather than an integer - benchmark.Add("Measurements", - report.AllMeasurements.Select(m => new - { - IterationMode = m.IterationMode.ToString(), - IterationStage = m.IterationStage.ToString(), - m.LaunchIndex, - m.IterationIndex, - m.Operations, - m.Nanoseconds - })); - - if (report.Metrics.Any()) - { - benchmark.Add("Metrics", report.Metrics.Values); - } + benchmark.Add("Metrics", report.Metrics.Values); } + } - return benchmark; + return benchmark; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Exporters/Json/SimpleJson.cs b/src/BenchmarkDotNet/Exporters/Json/SimpleJsonSerializer.cs similarity index 99% rename from src/BenchmarkDotNet/Exporters/Json/SimpleJson.cs rename to src/BenchmarkDotNet/Exporters/Json/SimpleJsonSerializer.cs index 5aa3ebe1c4..e461c8a5c6 100644 --- a/src/BenchmarkDotNet/Exporters/Json/SimpleJson.cs +++ b/src/BenchmarkDotNet/Exporters/Json/SimpleJsonSerializer.cs @@ -1,3 +1,4 @@ +#nullable disable // ReSharper disable All //----------------------------------------------------------------------- // @@ -104,7 +105,7 @@ public JsonArray(int capacity) : base(capacity) { } /// The json representation of the array. public override string ToString() { - return SimpleJson.SerializeObject(this) ?? string.Empty; + return SimpleJsonSerializer.SerializeObject(this) ?? string.Empty; } } @@ -343,7 +344,7 @@ IEnumerator IEnumerable.GetEnumerator() /// public override string ToString() { - return SimpleJson.SerializeObject(this); + return SimpleJsonSerializer.SerializeObject(this); } #if SIMPLE_JSON_DYNAMIC @@ -500,7 +501,7 @@ namespace SimpleJson #else public #endif - static class SimpleJson + static class SimpleJsonSerializer { private const int TOKEN_NONE = 0; private const int TOKEN_CURLY_OPEN = 1; @@ -521,7 +522,7 @@ static class SimpleJson private static readonly char[] EscapeCharacters = new char[] { '"', '\\', '\b', '\f', '\n', '\r', '\t' }; private static readonly string EscapeCharactersString = new string(EscapeCharacters); - static SimpleJson() + static SimpleJsonSerializer() { EscapeTable = new char[93]; EscapeTable['"'] = '"'; diff --git a/src/BenchmarkDotNet/Exporters/MarkdownExporter.cs b/src/BenchmarkDotNet/Exporters/MarkdownExporter.cs index 668396baa8..2a106e0132 100644 --- a/src/BenchmarkDotNet/Exporters/MarkdownExporter.cs +++ b/src/BenchmarkDotNet/Exporters/MarkdownExporter.cs @@ -164,9 +164,9 @@ private void PrintTable(SummaryTable table, ILogger logger) logger.WriteStatistic(ColumnsStartWithSeparator ? TableHeaderSeparator.TrimStart().TrimEnd() + "-" : "-"); logger.WriteLineStatistic(string.Join("", - table.Columns.Where(c => c.NeedToShow).Select(column => + table.Columns.Where(c => c.NeedToShow).Select((column, index) => new string('-', column.Width - 1) + GetHeaderSeparatorIndicator(column.OriginalColumn.IsNumeric) + - GetHeaderSeparatorColumnDivider(column.Index, table.ColumnCount)))); + GetHeaderSeparatorColumnDivider(index, table.Columns.Where(c => c.NeedToShow).Count())))); } int rowCounter = 0; diff --git a/src/BenchmarkDotNet/Exporters/PerfonarJsonExporter.cs b/src/BenchmarkDotNet/Exporters/PerfonarJsonExporter.cs new file mode 100644 index 0000000000..ce8287fb2d --- /dev/null +++ b/src/BenchmarkDotNet/Exporters/PerfonarJsonExporter.cs @@ -0,0 +1,18 @@ +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using Perfolizer.Json; + +namespace BenchmarkDotNet.Exporters; + +/// +/// IMPORTANT: Not fully implemented yet +/// +internal class PerfonarJsonExporter(LightJsonSettings? jsonSettings = null) : ExporterBase +{ + protected override string FileExtension => "perfonar.json"; + + public override void ExportToLog(Summary summary, ILogger logger) + { + logger.WriteLine(LightJsonSerializer.Serialize(summary.ToPerfonar(), jsonSettings)); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Exporters/PerfonarMdExporter.cs b/src/BenchmarkDotNet/Exporters/PerfonarMdExporter.cs new file mode 100644 index 0000000000..d2d4e77a79 --- /dev/null +++ b/src/BenchmarkDotNet/Exporters/PerfonarMdExporter.cs @@ -0,0 +1,21 @@ +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; + +using Perfolizer.Perfonar.Tables; + +namespace BenchmarkDotNet.Exporters; + +/// +/// IMPORTANT: Not fully implemented yet +/// +internal class PerfonarMdExporter : ExporterBase +{ + protected override string FileExtension => "perfonar.md"; + + public override void ExportToLog(Summary summary, ILogger logger) + { + var table = new PerfonarTable(summary.ToPerfonar(), summary.GetDefaultTableConfig()); + string markdown = table.ToMarkdown(new PerfonarTableStyle()); + logger.WriteLine(markdown); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Exporters/RPlotExporter.cs b/src/BenchmarkDotNet/Exporters/RPlotExporter.cs index 5c23f3b598..c90338f490 100644 --- a/src/BenchmarkDotNet/Exporters/RPlotExporter.cs +++ b/src/BenchmarkDotNet/Exporters/RPlotExporter.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Exporters.Csv; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; @@ -82,7 +83,7 @@ public void ExportToLog(Summary summary, ILogger logger) private static bool TryFindRScript(ILogger consoleLogger, out string? rscriptPath) { - string rscriptExecutable = RuntimeInformation.IsWindows() ? "Rscript.exe" : "Rscript"; + string rscriptExecutable = OsDetector.IsWindows() ? "Rscript.exe" : "Rscript"; rscriptPath = null; string rHome = Environment.GetEnvironmentVariable("R_HOME"); @@ -101,7 +102,7 @@ private static bool TryFindRScript(ILogger consoleLogger, out string? rscriptPat if (rscriptPath != null) return true; - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) { string programFiles = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles); string programFilesR = Path.Combine(programFiles, "R"); diff --git a/src/BenchmarkDotNet/Exporters/Xml/SummaryDto.cs b/src/BenchmarkDotNet/Exporters/Xml/SummaryDto.cs index 13c9a74619..3087250b52 100644 --- a/src/BenchmarkDotNet/Exporters/Xml/SummaryDto.cs +++ b/src/BenchmarkDotNet/Exporters/Xml/SummaryDto.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Reports; using JetBrains.Annotations; +using Perfolizer.Helpers; using Perfolizer.Horology; // ReSharper disable UnusedMember.Global @@ -34,11 +35,11 @@ internal class HostEnvironmentInfoDto { public string BenchmarkDotNetCaption => HostEnvironmentInfo.BenchmarkDotNetCaption; public string BenchmarkDotNetVersion => hei.BenchmarkDotNetVersion; - public string OsVersion => hei.OsVersion.Value; - public string ProcessorName => ProcessorBrandStringHelper.Prettify(hei.CpuInfo.Value); - public string PhysicalProcessorCount => hei.CpuInfo.Value?.PhysicalProcessorCount?.ToString(); - public string PhysicalCoreCount => hei.CpuInfo.Value?.PhysicalCoreCount?.ToString(); - public string LogicalCoreCount => hei.CpuInfo.Value?.LogicalCoreCount?.ToString(); + public string OsVersion => hei.Os.Value.ToBrandString(); + public string ProcessorName => hei.Cpu.Value.ToShortBrandName(); + public string PhysicalProcessorCount => hei.Cpu.Value?.PhysicalProcessorCount?.ToString(); + public string PhysicalCoreCount => hei.Cpu.Value?.PhysicalCoreCount?.ToString(); + public string LogicalCoreCount => hei.Cpu.Value?.LogicalCoreCount?.ToString(); public string RuntimeVersion => hei.RuntimeVersion; public string Architecture => hei.Architecture; public bool HasAttachedDebugger => hei.HasAttachedDebugger; @@ -71,6 +72,8 @@ internal class BenchmarkReportDto public string MethodTitle => report.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo; public string Parameters => report.BenchmarkCase.Parameters.PrintInfo; public Statistics Statistics => report.ResultStatistics; + public IEnumerable Metrics => report.Metrics.Values; + public GcStats Memory => new GcStats() { Gen0Collections = report.GcStats.Gen0Collections, @@ -79,6 +82,7 @@ internal class BenchmarkReportDto TotalOperations = report.GcStats.TotalOperations, BytesAllocatedPerOperation = report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) }; + [PublicAPI] public IEnumerable Measurements { get; } private readonly BenchmarkReport report; @@ -104,4 +108,4 @@ internal struct GcStats public long TotalOperations { get; set; } public long? BytesAllocatedPerOperation { get; set; } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Extensions/CommonExtensions.cs b/src/BenchmarkDotNet/Extensions/CommonExtensions.cs index a753787c53..aa402ee5dd 100644 --- a/src/BenchmarkDotNet/Extensions/CommonExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/CommonExtensions.cs @@ -20,11 +20,11 @@ public static string GetColumnTitle(this IColumn column, SummaryStyle style) switch (column.UnitType) { case UnitType.CodeSize: - return $"{column.ColumnName} [{style.CodeSizeUnit.Name}]"; + return $"{column.ColumnName} [{style.CodeSizeUnit.Abbreviation}]"; case UnitType.Size: - return $"{column.ColumnName} [{style.SizeUnit.Name}]"; + return $"{column.ColumnName} [{style.SizeUnit.Abbreviation}]"; case UnitType.Time: - return $"{column.ColumnName} [{style.TimeUnit.Name}]"; + return $"{column.ColumnName} [{style.TimeUnit.Abbreviation}]"; case UnitType.Dimensionless: return column.ColumnName; default: @@ -32,10 +32,12 @@ public static string GetColumnTitle(this IColumn column, SummaryStyle style) } } - public static bool IsNullOrEmpty(this IReadOnlyCollection value) => value == null || value.Count == 0; + public static bool IsNullOrEmpty(this IReadOnlyCollection? value) => value == null || value.Count == 0; public static bool IsEmpty(this IReadOnlyCollection value) => value.Count == 0; public static bool IsEmpty(this IEnumerable value) => !value.Any(); + public static IEnumerable WhereNotNull(this IEnumerable values) => values.Where(value => value != null).Cast(); + public static void AddRange(this HashSet hashSet, IEnumerable collection) { foreach (var item in collection) @@ -43,7 +45,7 @@ public static void AddRange(this HashSet hashSet, IEnumerable collectio } #if NETSTANDARD2_0 - public static TValue GetValueOrDefault(this IDictionary dictionary, TKey key) + public static TValue? GetValueOrDefault(this IDictionary dictionary, TKey key) => dictionary.TryGetValue(key, out var value) ? value : default; #endif @@ -97,7 +99,9 @@ internal static string DeleteFileIfExists(this string filePath) internal static string EnsureFolderExists(this string filePath) { - string directoryPath = Path.GetDirectoryName(filePath); + string? directoryPath = Path.GetDirectoryName(filePath); + if (directoryPath == null) + throw new ArgumentException($"Can't get directory path from '{filePath}'"); if (!Directory.Exists(directoryPath)) Directory.CreateDirectory(directoryPath); @@ -105,7 +109,7 @@ internal static string EnsureFolderExists(this string filePath) return filePath; } - internal static bool IsNotNullButDoesNotExist(this FileSystemInfo fileInfo) + internal static bool IsNotNullButDoesNotExist(this FileSystemInfo? fileInfo) => fileInfo != null && !fileInfo.Exists; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Extensions/MathExtensions.cs b/src/BenchmarkDotNet/Extensions/MathExtensions.cs new file mode 100644 index 0000000000..86a17f93f5 --- /dev/null +++ b/src/BenchmarkDotNet/Extensions/MathExtensions.cs @@ -0,0 +1,9 @@ +using System; + +namespace BenchmarkDotNet.Extensions; + +internal static class MathExtensions +{ + public static int RoundToInt(this double x) => (int)Math.Round(x); + public static long RoundToLong(this double x) => (long)Math.Round(x); +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs index 912babeb39..4d0854ba03 100644 --- a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.IO; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; @@ -86,7 +87,7 @@ public static bool TrySetAffinity( if (logger == null) throw new ArgumentNullException(nameof(logger)); - if (!RuntimeInformation.IsWindows() && !RuntimeInformation.IsLinux()) + if (!OsDetector.IsWindows() && !OsDetector.IsLinux()) return false; try @@ -108,7 +109,7 @@ public static bool TrySetAffinity( if (process == null) throw new ArgumentNullException(nameof(process)); - if (!RuntimeInformation.IsWindows() && !RuntimeInformation.IsLinux()) + if (!OsDetector.IsWindows() && !OsDetector.IsLinux()) return null; try @@ -124,7 +125,7 @@ public static bool TrySetAffinity( internal static void SetEnvironmentVariables(this ProcessStartInfo start, BenchmarkCase benchmarkCase, IResolver resolver) { if (benchmarkCase.Job.Environment.Runtime is ClrRuntime clrRuntime && !string.IsNullOrEmpty(clrRuntime.Version)) - start.EnvironmentVariables["COMPLUS_Version"] = clrRuntime.Version; + SetClrEnvironmentVariables(start, "Version", clrRuntime.Version); if (benchmarkCase.Job.Environment.Runtime is MonoRuntime monoRuntime && !string.IsNullOrEmpty(monoRuntime.MonoBclPath)) start.EnvironmentVariables["MONO_PATH"] = monoRuntime.MonoBclPath; @@ -132,10 +133,10 @@ internal static void SetEnvironmentVariables(this ProcessStartInfo start, Benchm if (benchmarkCase.Config.HasPerfCollectProfiler()) { // enable tracing configuration inside of CoreCLR (https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/linux-performance-tracing.md#collecting-a-trace) - start.EnvironmentVariables["COMPlus_PerfMapEnabled"] = "1"; - start.EnvironmentVariables["COMPlus_EnableEventLog"] = "1"; + SetClrEnvironmentVariables(start, "PerfMapEnabled", "1"); + SetClrEnvironmentVariables(start, "EnableEventLog", "1"); // enable BDN Event Source (https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/linux-performance-tracing.md#filtering) - start.EnvironmentVariables["COMPlus_EventSourceFilter"] = EngineEventSource.SourceName; + SetClrEnvironmentVariables(start, "EventSourceFilter", EngineEventSource.SourceName); // workaround for https://github.com/dotnet/runtime/issues/71786, will be solved by next perf version start.EnvironmentVariables["DOTNET_EnableWriteXorExecute"] = "0"; } @@ -163,7 +164,7 @@ internal static void SetEnvironmentVariables(this ProcessStartInfo start, Benchm public static void KillTree(this Process process, TimeSpan timeout) { - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) { RunProcessAndIgnoreOutput("taskkill", $"/T /F /PID {process.Id}", timeout); } @@ -257,21 +258,27 @@ private static void SetCoreRunEnvironmentVariables(this ProcessStartInfo start, { var gcMode = benchmarkCase.Job.Environment.Gc; - start.EnvironmentVariables["COMPlus_gcServer"] = gcMode.ResolveValue(GcMode.ServerCharacteristic, resolver) ? "1" : "0"; - start.EnvironmentVariables["COMPlus_gcConcurrent"] = gcMode.ResolveValue(GcMode.ConcurrentCharacteristic, resolver) ? "1" : "0"; + SetClrEnvironmentVariables(start, "gcServer", gcMode.ResolveValue(GcMode.ServerCharacteristic, resolver) ? "1" : "0"); + SetClrEnvironmentVariables(start, "gcConcurrent", gcMode.ResolveValue(GcMode.ConcurrentCharacteristic, resolver) ? "1" : "0"); if (gcMode.HasValue(GcMode.CpuGroupsCharacteristic)) - start.EnvironmentVariables["COMPlus_GCCpuGroup"] = gcMode.ResolveValue(GcMode.CpuGroupsCharacteristic, resolver) ? "1" : "0"; + SetClrEnvironmentVariables(start, "GCCpuGroup", gcMode.ResolveValue(GcMode.CpuGroupsCharacteristic, resolver) ? "1" : "0"); if (gcMode.HasValue(GcMode.AllowVeryLargeObjectsCharacteristic)) - start.EnvironmentVariables["COMPlus_gcAllowVeryLargeObjects"] = gcMode.ResolveValue(GcMode.AllowVeryLargeObjectsCharacteristic, resolver) ? "1" : "0"; + SetClrEnvironmentVariables(start, "gcAllowVeryLargeObjects", gcMode.ResolveValue(GcMode.AllowVeryLargeObjectsCharacteristic, resolver) ? "1" : "0"); if (gcMode.HasValue(GcMode.RetainVmCharacteristic)) - start.EnvironmentVariables["COMPlus_GCRetainVM"] = gcMode.ResolveValue(GcMode.RetainVmCharacteristic, resolver) ? "1" : "0"; + SetClrEnvironmentVariables(start, "GCRetainVM", gcMode.ResolveValue(GcMode.RetainVmCharacteristic, resolver) ? "1" : "0"); if (gcMode.HasValue(GcMode.NoAffinitizeCharacteristic)) - start.EnvironmentVariables["COMPlus_GCNoAffinitize"] = gcMode.ResolveValue(GcMode.NoAffinitizeCharacteristic, resolver) ? "1" : "0"; + SetClrEnvironmentVariables(start, "GCNoAffinitize", gcMode.ResolveValue(GcMode.NoAffinitizeCharacteristic, resolver) ? "1" : "0"); if (gcMode.HasValue(GcMode.HeapAffinitizeMaskCharacteristic)) - start.EnvironmentVariables["COMPlus_GCHeapAffinitizeMask"] = gcMode.HeapAffinitizeMask.ToString("X"); + SetClrEnvironmentVariables(start, "GCHeapAffinitizeMask", gcMode.HeapAffinitizeMask.ToString("X")); if (gcMode.HasValue(GcMode.HeapCountCharacteristic)) - start.EnvironmentVariables["COMPlus_GCHeapCount"] = gcMode.HeapCount.ToString("X"); + SetClrEnvironmentVariables(start, "GCHeapCount", gcMode.HeapCount.ToString("X")); + } + + private static void SetClrEnvironmentVariables(ProcessStartInfo start, string suffix, string value) + { + start.EnvironmentVariables[$"DOTNET_{suffix}"] = value; + start.EnvironmentVariables[$"COMPlus_{suffix}"] = value; } } } diff --git a/src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs b/src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs index cf1f71a166..0a1a7c22fc 100644 --- a/src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs @@ -155,10 +155,7 @@ internal static bool ContainsRunnableBenchmarks(this Type type) { var typeInfo = type.GetTypeInfo(); - if (typeInfo.IsAbstract - || typeInfo.IsSealed - || typeInfo.IsNotPublic - || typeInfo.IsGenericType && !IsRunnableGenericType(typeInfo)) + if (typeInfo.IsAbstract || typeInfo.IsGenericType && !IsRunnableGenericType(typeInfo)) return false; return typeInfo.GetBenchmarks().Any(); diff --git a/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs b/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs index 5ba51eebdd..e3fbdb04e0 100644 --- a/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs @@ -47,6 +47,8 @@ internal static Runtime GetRuntime(this RuntimeMoniker runtimeMoniker) return CoreRuntime.Core80; case RuntimeMoniker.Net90: return CoreRuntime.Core90; + case RuntimeMoniker.Net10_0: + return CoreRuntime.Core10_0; case RuntimeMoniker.Mono: return MonoRuntime.Default; case RuntimeMoniker.NativeAot60: @@ -57,6 +59,8 @@ internal static Runtime GetRuntime(this RuntimeMoniker runtimeMoniker) return NativeAotRuntime.Net80; case RuntimeMoniker.NativeAot90: return NativeAotRuntime.Net90; + case RuntimeMoniker.NativeAot10_0: + return NativeAotRuntime.Net10_0; case RuntimeMoniker.Mono60: return MonoRuntime.Mono60; case RuntimeMoniker.Mono70: @@ -65,9 +69,57 @@ internal static Runtime GetRuntime(this RuntimeMoniker runtimeMoniker) return MonoRuntime.Mono80; case RuntimeMoniker.Mono90: return MonoRuntime.Mono90; + case RuntimeMoniker.Mono10_0: + return MonoRuntime.Mono10_0; default: throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, "Runtime Moniker not supported"); } } + + internal static Version GetRuntimeVersion(this RuntimeMoniker runtimeMoniker) => runtimeMoniker switch + { + RuntimeMoniker.Net461 => new Version(4, 6, 1), + RuntimeMoniker.Net462 => new Version(4, 6, 2), + RuntimeMoniker.Net47 => new Version(4, 7), + RuntimeMoniker.Net471 => new Version(4, 7, 1), + RuntimeMoniker.Net472 => new Version(4, 7, 2), + RuntimeMoniker.Net48 => new Version(4, 8), + RuntimeMoniker.Net481 => new Version(4, 8, 1), + RuntimeMoniker.NetCoreApp20 => new Version(2, 0), + RuntimeMoniker.NetCoreApp21 => new Version(2, 1), + RuntimeMoniker.NetCoreApp22 => new Version(2, 2), + RuntimeMoniker.NetCoreApp30 => new Version(3, 0), + RuntimeMoniker.NetCoreApp31 => new Version(3, 1), + RuntimeMoniker.Net50 => new Version(5, 0), + RuntimeMoniker.Net60 => new Version(6, 0), + RuntimeMoniker.Net70 => new Version(7, 0), + RuntimeMoniker.Net80 => new Version(8, 0), + RuntimeMoniker.Net90 => new Version(9, 0), + RuntimeMoniker.Net10_0 => new Version(10, 0), + RuntimeMoniker.NativeAot60 => new Version(6, 0), + RuntimeMoniker.NativeAot70 => new Version(7, 0), + RuntimeMoniker.NativeAot80 => new Version(8, 0), + RuntimeMoniker.NativeAot90 => new Version(9, 0), + RuntimeMoniker.NativeAot10_0 => new Version(10, 0), + RuntimeMoniker.Mono60 => new Version(6, 0), + RuntimeMoniker.Mono70 => new Version(7, 0), + RuntimeMoniker.Mono80 => new Version(8, 0), + RuntimeMoniker.Mono90 => new Version(9, 0), + RuntimeMoniker.Mono10_0 => new Version(10, 0), + RuntimeMoniker.Wasm => Portability.RuntimeInformation.IsNetCore && CoreRuntime.TryGetVersion(out var version) ? version : new Version(5, 0), + RuntimeMoniker.WasmNet50 => new Version(5, 0), + RuntimeMoniker.WasmNet60 => new Version(6, 0), + RuntimeMoniker.WasmNet70 => new Version(7, 0), + RuntimeMoniker.WasmNet80 => new Version(8, 0), + RuntimeMoniker.WasmNet90 => new Version(9, 0), + RuntimeMoniker.WasmNet10_0 => new Version(10, 0), + RuntimeMoniker.MonoAOTLLVM => Portability.RuntimeInformation.IsNetCore && CoreRuntime.TryGetVersion(out var version) ? version : new Version(6, 0), + RuntimeMoniker.MonoAOTLLVMNet60 => new Version(6, 0), + RuntimeMoniker.MonoAOTLLVMNet70 => new Version(7, 0), + RuntimeMoniker.MonoAOTLLVMNet80 => new Version(8, 0), + RuntimeMoniker.MonoAOTLLVMNet90 => new Version(9, 0), + RuntimeMoniker.MonoAOTLLVMNet10_0 => new Version(10, 0), + _ => throw new NotImplementedException($"{nameof(GetRuntimeVersion)} not implemented for {runtimeMoniker}") + }; } } diff --git a/src/BenchmarkDotNet/Extensions/StatisticsExtensions.cs b/src/BenchmarkDotNet/Extensions/StatisticsExtensions.cs index 7abb00768b..922dc50d44 100644 --- a/src/BenchmarkDotNet/Extensions/StatisticsExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/StatisticsExtensions.cs @@ -1,6 +1,7 @@ using System; using System.Globalization; using System.Text; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Mathematics; using JetBrains.Annotations; using Perfolizer.Horology; @@ -16,11 +17,11 @@ public static class StatisticsExtensions public static Func CreateNanosecondFormatter(this Statistics s, CultureInfo cultureInfo, string format = "N3") { var timeUnit = TimeUnit.GetBestTimeUnit(s.Mean); - return x => TimeInterval.FromNanoseconds(x).ToString(timeUnit, cultureInfo, format); + return x => TimeInterval.FromNanoseconds(x).ToString(timeUnit, format, cultureInfo, UnitHelper.DefaultPresentation); } [PublicAPI] - public static string ToString(this Statistics s, CultureInfo cultureInfo, Func formatter, bool calcHistogram = false) + public static string ToString(this Statistics? s, CultureInfo cultureInfo, Func formatter, bool calcHistogram = false) { if (s == null) return NullSummaryMessage; @@ -29,9 +30,9 @@ public static string ToString(this Statistics s, CultureInfo cultureInfo, Func> InvalidFileNameCharactersMappings = new Lazy>(BuildInvalidPathCharactersMappings); @@ -73,7 +73,7 @@ private static Dictionary BuildInvalidPathCharactersMappings() return invalidFileNameChars.ToDictionary( character => character.ToString(), - character => $"char{(short) character}"); + character => $"char{(short)character}"); } /// @@ -150,5 +150,8 @@ internal static StringBuilder AppendArgument(this StringBuilder stringBuilder, s /// The string builder with the arguments added internal static StringBuilder AppendArgument(this StringBuilder stringBuilder, object argument) => argument == null ? stringBuilder : AppendArgument(stringBuilder, argument.ToString()); + + public static bool IsBlank(this string? value) => string.IsNullOrWhiteSpace(value); + public static bool IsNotBlank(this string? value) => !value.IsBlank(); } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs b/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs index 62b4d31d94..d98ca46e58 100644 --- a/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs +++ b/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Linq; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; @@ -17,7 +18,7 @@ internal static class ArtifactFileNameHelper internal static string GetTraceFilePath(DiagnoserActionParameters details, DateTime creationTime, string fileExtension) { - return GetFilePath(details, null, creationTime, fileExtension, "userheap.etl".Length); + return GetFilePath(details, null, creationTime, fileExtension, "userheap.etl".Length - fileExtension.Length); } internal static string GetFilePath(DiagnoserActionParameters details, string? subfolder, DateTime? creationTime, string fileExtension, int reserve) @@ -26,7 +27,7 @@ internal static string GetFilePath(DiagnoserActionParameters details, string? su // long paths can be enabled on Windows but it does not mean that everything is going to work fine.. // so we always use 260 as limit on Windows - int limit = RuntimeInformation.IsWindows() + int limit = OsDetector.IsWindows() ? WindowsOldPathLimit - reserve : CommonSenseLimit; @@ -68,7 +69,7 @@ private static string GetLimitedFilePath(DiagnoserActionParameters details, stri private static string GetFilePath(string fileName, DiagnoserActionParameters details, string? subfolder, DateTime? creationTime, string fileExtension) { - // if we run for more than one toolchain, the output file name should contain the name too so we can differ net462 vs netcoreapp2.1 etc + // if we run for more than one toolchain, the output file name should contain the name too so we can differ net462 vs net8.0 etc if (details.Config.GetJobs().Select(job => ToolchainExtensions.GetToolchain(job)).Distinct().Count() > 1) fileName += $"-{details.BenchmarkCase.Job.Environment.Runtime?.Name ?? details.BenchmarkCase.GetToolchain()?.Name ?? details.BenchmarkCase.Job.Id}"; diff --git a/src/BenchmarkDotNet/Helpers/Assertion.cs b/src/BenchmarkDotNet/Helpers/Assertion.cs new file mode 100644 index 0000000000..44e8b36e46 --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/Assertion.cs @@ -0,0 +1,14 @@ +using System; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Helpers; + +internal static class Assertion +{ + [AssertionMethod] + public static void NotNull(string name, object? value) + { + if (value == null) + throw new ArgumentNullException(name, $"{name} can't be null"); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/AwaitHelper.cs b/src/BenchmarkDotNet/Helpers/AwaitHelper.cs new file mode 100644 index 0000000000..98837a7ea6 --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/AwaitHelper.cs @@ -0,0 +1,108 @@ +using System; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace BenchmarkDotNet.Helpers +{ + public static class AwaitHelper + { + private class ValueTaskWaiter + { + // We use thread static field so that each thread uses its own individual callback and reset event. + [ThreadStatic] + private static ValueTaskWaiter ts_current; + internal static ValueTaskWaiter Current => ts_current ??= new ValueTaskWaiter(); + + // We cache the callback to prevent allocations for memory diagnoser. + private readonly Action awaiterCallback; + private readonly ManualResetEventSlim resetEvent; + + private ValueTaskWaiter() + { + resetEvent = new(); + awaiterCallback = resetEvent.Set; + } + + internal void Wait(TAwaiter awaiter) where TAwaiter : ICriticalNotifyCompletion + { + resetEvent.Reset(); + awaiter.UnsafeOnCompleted(awaiterCallback); + + // The fastest way to wait for completion is to spin a bit before waiting on the event. This is the same logic that Task.GetAwaiter().GetResult() uses. + var spinner = new SpinWait(); + while (!resetEvent.IsSet) + { + if (spinner.NextSpinWillYield) + { + resetEvent.Wait(); + return; + } + spinner.SpinOnce(); + } + } + } + + // we use GetAwaiter().GetResult() because it's fastest way to obtain the result in blocking way, + // and will eventually throw actual exception, not aggregated one + public static void GetResult(Task task) => task.GetAwaiter().GetResult(); + + public static T GetResult(Task task) => task.GetAwaiter().GetResult(); + + // ValueTask can be backed by an IValueTaskSource that only supports asynchronous awaits, + // so we have to hook up a callback instead of calling .GetAwaiter().GetResult() like we do for Task. + // The alternative is to convert it to Task using .AsTask(), but that causes allocations which we must avoid for memory diagnoser. + public static void GetResult(ValueTask task) + { + // Don't continue on the captured context, as that may result in a deadlock if the user runs this in-process. + var awaiter = task.ConfigureAwait(false).GetAwaiter(); + if (!awaiter.IsCompleted) + { + ValueTaskWaiter.Current.Wait(awaiter); + } + awaiter.GetResult(); + } + + public static T GetResult(ValueTask task) + { + // Don't continue on the captured context, as that may result in a deadlock if the user runs this in-process. + var awaiter = task.ConfigureAwait(false).GetAwaiter(); + if (!awaiter.IsCompleted) + { + ValueTaskWaiter.Current.Wait(awaiter); + } + return awaiter.GetResult(); + } + + internal static MethodInfo GetGetResultMethod(Type taskType) + { + if (!taskType.IsGenericType) + { + return typeof(AwaitHelper).GetMethod(nameof(AwaitHelper.GetResult), BindingFlags.Public | BindingFlags.Static, null, new Type[1] { taskType }, null); + } + + Type compareType = taskType.GetGenericTypeDefinition() == typeof(ValueTask<>) ? typeof(ValueTask<>) + : typeof(Task).IsAssignableFrom(taskType.GetGenericTypeDefinition()) ? typeof(Task<>) + : null; + if (compareType == null) + { + return null; + } + var resultType = taskType + .GetMethod(nameof(Task.GetAwaiter), BindingFlags.Public | BindingFlags.Instance) + .ReturnType + .GetMethod(nameof(TaskAwaiter.GetResult), BindingFlags.Public | BindingFlags.Instance) + .ReturnType; + return typeof(AwaitHelper).GetMethods(BindingFlags.Public | BindingFlags.Static) + .First(m => + { + if (m.Name != nameof(AwaitHelper.GetResult)) return false; + Type paramType = m.GetParameters().First().ParameterType; + return paramType.IsGenericType && paramType.GetGenericTypeDefinition() == compareType; + }) + .MakeGenericMethod(new[] { resultType }); + } + } +} diff --git a/src/BenchmarkDotNet/Helpers/DirtyAssemblyResolveHelper.cs b/src/BenchmarkDotNet/Helpers/DirtyAssemblyResolveHelper.cs index a16e15f653..0921b7d1d6 100644 --- a/src/BenchmarkDotNet/Helpers/DirtyAssemblyResolveHelper.cs +++ b/src/BenchmarkDotNet/Helpers/DirtyAssemblyResolveHelper.cs @@ -28,7 +28,7 @@ internal class DirtyAssemblyResolveHelper : IDisposable /// "the handler is invoked whenever the runtime fails to bind to an assembly by name." /// /// not null when we find it manually, null when can't help - private Assembly HelpTheFrameworkToResolveTheAssembly(object sender, ResolveEventArgs args) + private Assembly? HelpTheFrameworkToResolveTheAssembly(object sender, ResolveEventArgs args) { var fullName = new AssemblyName(args.Name); string simpleName = fullName.Name; diff --git a/src/BenchmarkDotNet/Helpers/DisposeAtProcessTermination.cs b/src/BenchmarkDotNet/Helpers/DisposeAtProcessTermination.cs new file mode 100644 index 0000000000..bc627a4a1b --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/DisposeAtProcessTermination.cs @@ -0,0 +1,70 @@ +using BenchmarkDotNet.Detectors; +using System; +using System.Runtime.InteropServices; + +namespace BenchmarkDotNet.Helpers +{ + /// + /// Ensures that explicit Dispose is called at termination of the Process. + /// + /// + /// + /// This class exists to help in reverting system state where C#'s using statement does not + /// suffice. I.e. when Benchmark's process is aborted via Ctrl-C, Ctrl-Break or via click on the + /// X in the upper right of Window. + /// + /// + /// Usage: Derive your clas that changes system state of this class. Revert system state in + /// override of implementation. + /// Use your class in C#'s using statement, to ensure system state is reverted in normal situations. + /// This class ensures your override is also called at process 'abort'. + /// + /// + /// Note: This class is explicitly not responsible for cleanup of Native resources. Of course, + /// derived classes can cleanup their Native resources (usually managed via + /// derived classes), by delegating explicit Disposal to their + /// fields. + /// + /// + public abstract class DisposeAtProcessTermination : IDisposable + { + private static readonly bool ConsoleSupportsCancelKeyPress + = !(OsDetector.IsAndroid() || OsDetector.IsIOS() || OsDetector.IsTvOS() || Portability.RuntimeInformation.IsWasm); + + protected DisposeAtProcessTermination() + { + AppDomain.CurrentDomain.ProcessExit += OnProcessExit; + if (ConsoleSupportsCancelKeyPress) + { + // Cancel key presses are not supported by .NET or do not exist for these + Console.CancelKeyPress += OnCancelKeyPress; + } + + // It does not make sense to include a Finalizer. We do not manage any native resource and: + // as we are subscribed to static events, it would never be called. + } + + /// + /// Called when the user presses Ctrl-C or Ctrl-Break. + /// + private void OnCancelKeyPress(object? sender, ConsoleCancelEventArgs e) + { + if (!e.Cancel) { Dispose(); } + } + + /// + /// Called when the user clicks on the X in the upper right corner to close the Benchmark's Window. + /// + private void OnProcessExit(object? sender, EventArgs e) => Dispose(); + + public virtual void Dispose() + { + AppDomain.CurrentDomain.ProcessExit -= OnProcessExit; + if (ConsoleSupportsCancelKeyPress) + { + // Cancel key presses are not supported by .NET or do not exist for these + Console.CancelKeyPress -= OnCancelKeyPress; + } + } + } +} diff --git a/src/BenchmarkDotNet/Helpers/ExternalToolsHelper.cs b/src/BenchmarkDotNet/Helpers/ExternalToolsHelper.cs index 3802608047..bd5274c6b3 100644 --- a/src/BenchmarkDotNet/Helpers/ExternalToolsHelper.cs +++ b/src/BenchmarkDotNet/Helpers/ExternalToolsHelper.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Helpers @@ -11,7 +12,7 @@ public static class ExternalToolsHelper /// MacOSX only. /// public static readonly Lazy> MacSystemProfilerData = - LazyParse(RuntimeInformation.IsMacOS, "system_profiler", "SPSoftwareDataType", s => SectionsHelper.ParseSection(s, ':')); + LazyParse(OsDetector.IsMacOS, "system_profiler", "SPSoftwareDataType", s => SectionsHelper.ParseSection(s, ':')); private static Lazy LazyParse(Func isAvailable, string fileName, string arguments, Func parseFunc) { diff --git a/src/BenchmarkDotNet/Helpers/PowerShellLocator.cs b/src/BenchmarkDotNet/Helpers/PowerShellLocator.cs new file mode 100644 index 0000000000..28b8e910f7 --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/PowerShellLocator.cs @@ -0,0 +1,81 @@ +using System; +using System.IO; +using System.Linq; +using System.Runtime.Versioning; +using System.Text.RegularExpressions; +using BenchmarkDotNet.Detectors; + +namespace BenchmarkDotNet.Helpers; + +/// +/// Locates PowerShell on a system, currently only supports on Windows. +/// +internal class PowerShellLocator +{ + private static readonly string WindowsPowershellPath = + $"{Environment.SystemDirectory}{Path.DirectorySeparatorChar}WindowsPowerShell{Path.DirectorySeparatorChar}" + + $"v1.0{Path.DirectorySeparatorChar}powershell.exe"; + +#if NET6_0_OR_GREATER + [SupportedOSPlatform("windows")] +#endif + internal static string? LocateOnWindows() + { + if (OsDetector.IsWindows() == false) + return null; + + string powershellPath; + + try + { + string programFiles = Environment.Is64BitOperatingSystem + ? Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles) + : Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86); + + string powershell7PlusPath = $"{programFiles}{Path.DirectorySeparatorChar}Powershell{Path.DirectorySeparatorChar}"; + + bool checkForPowershell7Plus = true; + + if (Directory.Exists(powershell7PlusPath)) + { + string[] subDirectories = Directory.EnumerateDirectories(powershell7PlusPath, "*", SearchOption.AllDirectories) + .ToArray(); + + //Use the highest number string directory for PowerShell so that we get the newest major PowerShell version + // Example version directories are 6, 7, and in the future 8. + string? subDirectory = subDirectories.Where(x => Regex.IsMatch(x, "[0-9]")) + .Select(x => x) + .OrderByDescending(x => x) + .FirstOrDefault(); + + if (subDirectory is not null) + { + powershell7PlusPath = $"{subDirectory}{Path.DirectorySeparatorChar}pwsh.exe"; + } + else + { + checkForPowershell7Plus = false; + } + } + + // Optimistically, use Cross-platform new PowerShell when available but fallback to Windows PowerShell if not available. + if (checkForPowershell7Plus) + { + powershellPath = File.Exists(powershell7PlusPath) ? powershell7PlusPath : WindowsPowershellPath; + } + else + { + powershellPath = WindowsPowershellPath; + } + + if (File.Exists(powershellPath) == false) + powershellPath = "PowerShell"; + } + catch + { + powershellPath = "PowerShell"; + } + + return powershellPath; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/ProcessHelper.cs b/src/BenchmarkDotNet/Helpers/ProcessHelper.cs index 1b75c6392f..caa3410b21 100644 --- a/src/BenchmarkDotNet/Helpers/ProcessHelper.cs +++ b/src/BenchmarkDotNet/Helpers/ProcessHelper.cs @@ -12,7 +12,8 @@ internal static class ProcessHelper /// Run external process and return the console output. /// In the case of any exception, null will be returned. /// - internal static string? RunAndReadOutput(string fileName, string arguments = "", ILogger? logger = null) + internal static string? RunAndReadOutput(string fileName, string arguments = "", ILogger? logger = null, + Dictionary? environmentVariables = null) { var processStartInfo = new ProcessStartInfo { @@ -24,6 +25,9 @@ internal static class ProcessHelper RedirectStandardOutput = true, RedirectStandardError = true }; + if (environmentVariables != null) + foreach (var variable in environmentVariables) + processStartInfo.Environment[variable.Key] = variable.Value; using (var process = new Process { StartInfo = processStartInfo }) using (new ConsoleExitHandler(process, logger ?? NullLogger.Instance)) { diff --git a/src/BenchmarkDotNet/Helpers/SectionsHelper.cs b/src/BenchmarkDotNet/Helpers/SectionsHelper.cs index 15c54fd8f9..e16442d0fb 100644 --- a/src/BenchmarkDotNet/Helpers/SectionsHelper.cs +++ b/src/BenchmarkDotNet/Helpers/SectionsHelper.cs @@ -32,5 +32,14 @@ public static List> ParseSections(string? content, ch .Where(s => s.Count > 0) .ToList(); } + + public static List> ParseSectionsForPowershellWmi(string? content, char separator) + { + return + Regex.Split(content ?? "", "(\r*\n)") + .Select(s => ParseSection(s, separator)) + .Where(s => s.Count > 0) + .ToList(); + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/Taskbar/TaskbarProgress.cs b/src/BenchmarkDotNet/Helpers/Taskbar/TaskbarProgress.cs deleted file mode 100644 index 19ad5f6ffe..0000000000 --- a/src/BenchmarkDotNet/Helpers/Taskbar/TaskbarProgress.cs +++ /dev/null @@ -1,197 +0,0 @@ -using System; -using System.Runtime.InteropServices; - -namespace BenchmarkDotNet.Helpers -{ - internal class TaskbarProgress : IDisposable - { - private static readonly bool OsVersionIsSupported = Portability.RuntimeInformation.IsWindows() - // Must be windows 7 or greater - && Environment.OSVersion.Version >= new Version(6, 1); - - private IntPtr consoleWindowHandle = IntPtr.Zero; - private IntPtr consoleHandle = IntPtr.Zero; - - [DllImport("kernel32.dll")] - private static extern IntPtr GetConsoleWindow(); - [DllImport("kernel32.dll", SetLastError = true)] - private static extern IntPtr GetStdHandle(int nStdHandle); - - private const int STD_OUTPUT_HANDLE = -11; - - internal TaskbarProgress() - { - if (OsVersionIsSupported) - { - consoleWindowHandle = GetConsoleWindow(); - consoleHandle = GetStdHandle(STD_OUTPUT_HANDLE); - Console.CancelKeyPress += OnConsoleCancelEvent; - } - } - - internal void SetState(TaskbarProgressState state) - { - if (OsVersionIsSupported) - { - TaskbarProgressCom.SetState(consoleWindowHandle, consoleHandle, state); - } - } - - internal void SetProgress(float progressValue) - { - if (OsVersionIsSupported) - { - TaskbarProgressCom.SetValue(consoleWindowHandle, consoleHandle, progressValue); - } - } - - private void OnConsoleCancelEvent(object sender, ConsoleCancelEventArgs e) - { - Dispose(); - } - - public void Dispose() - { - if (OsVersionIsSupported) - { - TaskbarProgressCom.SetState(consoleWindowHandle, consoleHandle, TaskbarProgressState.NoProgress); - consoleWindowHandle = IntPtr.Zero; - consoleHandle = IntPtr.Zero; - Console.CancelKeyPress -= OnConsoleCancelEvent; - } - } - } - - internal enum TaskbarProgressState - { - NoProgress = 0, - Indeterminate = 0x1, - Normal = 0x2, - Error = 0x4, - Paused = 0x8, - Warning = Paused - } - - internal static class TaskbarProgressCom - { - [DllImport("kernel32.dll", SetLastError = true)] - private static extern bool GetConsoleMode(IntPtr hConsoleHandle, out ConsoleModes lpMode); - [DllImport("kernel32.dll", SetLastError = true)] - private static extern bool SetConsoleMode(IntPtr hConsoleHandle, ConsoleModes dwMode); - - [Flags] - private enum ConsoleModes : uint - { - ENABLE_PROCESSED_INPUT = 0x0001, - ENABLE_LINE_INPUT = 0x0002, - ENABLE_ECHO_INPUT = 0x0004, - ENABLE_WINDOW_INPUT = 0x0008, - ENABLE_MOUSE_INPUT = 0x0010, - ENABLE_INSERT_MODE = 0x0020, - ENABLE_QUICK_EDIT_MODE = 0x0040, - ENABLE_EXTENDED_FLAGS = 0x0080, - ENABLE_AUTO_POSITION = 0x0100, - - ENABLE_PROCESSED_OUTPUT = 0x0001, - ENABLE_WRAP_AT_EOL_OUTPUT = 0x0002, - ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004, - DISABLE_NEWLINE_AUTO_RETURN = 0x0008, - ENABLE_LVB_GRID_WORLDWIDE = 0x0010 - } - - [ComImport] - [Guid("ea1afb91-9e28-4b86-90e9-9e9f8a5eefaf")] - [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - private interface ITaskbarList3 - { - // ITaskbarList - [PreserveSig] - void HrInit(); - [PreserveSig] - void AddTab(IntPtr hwnd); - [PreserveSig] - void DeleteTab(IntPtr hwnd); - [PreserveSig] - void ActivateTab(IntPtr hwnd); - [PreserveSig] - void SetActiveAlt(IntPtr hwnd); - - // ITaskbarList2 - [PreserveSig] - void MarkFullscreenWindow(IntPtr hwnd, [MarshalAs(UnmanagedType.Bool)] bool fFullscreen); - - // ITaskbarList3 - [PreserveSig] - void SetProgressValue(IntPtr hwnd, ulong ullCompleted, ulong ullTotal); - [PreserveSig] - void SetProgressState(IntPtr hwnd, TaskbarProgressState state); - } - - [Guid("56FDF344-FD6D-11d0-958A-006097C9A090")] - [ClassInterface(ClassInterfaceType.None)] - [ComImport] - private class TaskbarInstance - { - } - - private static readonly ITaskbarList3 s_taskbarInstance = (ITaskbarList3) new TaskbarInstance(); - - internal static void SetState(IntPtr consoleWindowHandle, IntPtr consoleHandle, TaskbarProgressState taskbarState) - { - if (consoleWindowHandle != IntPtr.Zero) - { - s_taskbarInstance.SetProgressState(consoleWindowHandle, taskbarState); - } - - if (consoleHandle != IntPtr.Zero) - { - // Write progress state to console for Windows Terminal (https://github.com/microsoft/terminal/issues/6700). - GetConsoleMode(consoleHandle, out ConsoleModes previousConsoleMode); - SetConsoleMode(consoleHandle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT); - switch (taskbarState) - { - case TaskbarProgressState.NoProgress: - Console.Write("\x1b]9;4;0;0\x1b\\"); - break; - case TaskbarProgressState.Indeterminate: - Console.Write("\x1b]9;4;3;0\x1b\\"); - break; - case TaskbarProgressState.Normal: - // Do nothing, this is set automatically when SetValue is called (and WT has no documented way to set this). - break; - case TaskbarProgressState.Error: - Console.Write("\x1b]9;4;2;0\x1b\\"); - break; - case TaskbarProgressState.Warning: - Console.Write("\x1b]9;4;4;0\x1b\\"); - break; - } - SetConsoleMode(consoleHandle, previousConsoleMode); - } - } - - internal static void SetValue(IntPtr consoleWindowHandle, IntPtr consoleHandle, float progressValue) - { - bool isValidRange = progressValue >= 0 & progressValue <= 1; - if (!isValidRange) - { - throw new ArgumentOutOfRangeException(nameof(progressValue), "progressValue must be between 0 and 1 inclusive."); - } - uint value = (uint) (progressValue * 100); - - if (consoleWindowHandle != IntPtr.Zero) - { - s_taskbarInstance.SetProgressValue(consoleWindowHandle, value, 100); - } - - if (consoleHandle != IntPtr.Zero) - { - // Write progress sequence to console for Windows Terminal (https://github.com/microsoft/terminal/discussions/14268). - GetConsoleMode(consoleHandle, out ConsoleModes previousConsoleMode); - SetConsoleMode(consoleHandle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT); - Console.Write($"\x1b]9;4;1;{value}\x1b\\"); - SetConsoleMode(consoleHandle, previousConsoleMode); - } - } - } -} diff --git a/src/BenchmarkDotNet/Helpers/TaskbarProgress.cs b/src/BenchmarkDotNet/Helpers/TaskbarProgress.cs new file mode 100644 index 0000000000..e3910a0924 --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/TaskbarProgress.cs @@ -0,0 +1,252 @@ +using System; +using System.Runtime.InteropServices; +using BenchmarkDotNet.Detectors; + +namespace BenchmarkDotNet.Helpers +{ + internal enum TaskbarProgressState + { + NoProgress = 0, + Indeterminate = 0x1, + Normal = 0x2, + Error = 0x4, + Paused = 0x8, + Warning = Paused + } + + internal class TaskbarProgress : DisposeAtProcessTermination + { + private static readonly bool OsVersionIsSupported = OsDetector.IsWindows() + // Must be windows 7 or greater + && Environment.OSVersion.Version >= new Version(6, 1); + + private Com? com; + private Terminal? terminal; + + private bool IsEnabled => com != null || terminal != null; + + internal TaskbarProgress(TaskbarProgressState initialTaskbarState) + { + if (OsVersionIsSupported) + { + com = Com.MaybeCreateInstanceAndSetInitialState(initialTaskbarState); + terminal = Terminal.MaybeCreateInstanceAndSetInitialState(initialTaskbarState); + } + } + + internal void SetState(TaskbarProgressState state) + { + com?.SetState(state); + terminal?.SetState(state); + } + + internal void SetProgress(float progressValue) + { + bool isValidRange = progressValue >= 0 & progressValue <= 1; + if (!isValidRange) + { + throw new ArgumentOutOfRangeException(nameof(progressValue), "progressValue must be between 0 and 1 inclusive."); + } + uint value = (uint)(progressValue * 100); + com?.SetValue(value); + terminal?.SetValue(value); + } + + public override void Dispose() + { + if (IsEnabled) + { + SetState(TaskbarProgressState.NoProgress); + com = null; + terminal = null; + } + base.Dispose(); + } + + private sealed class Com + { + [ComImport] + [Guid("ea1afb91-9e28-4b86-90e9-9e9f8a5eefaf")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + private interface ITaskbarList3 + { + // ITaskbarList + [PreserveSig] + void HrInit(); + [PreserveSig] + void AddTab(IntPtr hwnd); + [PreserveSig] + void DeleteTab(IntPtr hwnd); + [PreserveSig] + void ActivateTab(IntPtr hwnd); + [PreserveSig] + void SetActiveAlt(IntPtr hwnd); + + // ITaskbarList2 + [PreserveSig] + void MarkFullscreenWindow(IntPtr hwnd, [MarshalAs(UnmanagedType.Bool)] bool fFullscreen); + + // ITaskbarList3 + [PreserveSig] + void SetProgressValue(IntPtr hwnd, ulong ullCompleted, ulong ullTotal); + [PreserveSig] + void SetProgressState(IntPtr hwnd, TaskbarProgressState state); + } + + [Guid("56FDF344-FD6D-11d0-958A-006097C9A090")] + [ClassInterface(ClassInterfaceType.None)] + [ComImport] + private class TaskbarInstance { } + + [DllImport("kernel32.dll")] + private static extern IntPtr GetConsoleWindow(); + + private readonly ITaskbarList3 taskbarInstance; + private readonly IntPtr consoleWindowHandle; + + private Com(IntPtr handle) + { + taskbarInstance = (ITaskbarList3) new TaskbarInstance(); + consoleWindowHandle = handle; + } + + internal static Com? MaybeCreateInstanceAndSetInitialState(TaskbarProgressState initialTaskbarState) + { + try + { + IntPtr handle = GetConsoleWindow(); + if (handle == IntPtr.Zero) + { + return null; + } + var com = new Com(handle); + com.SetState(initialTaskbarState); + return com; + } + // COM may be disabled, in which case this will throw (#2253). + // It could be NotSupportedException or COMException, we just catch all. + catch + { + return null; + } + } + + internal void SetState(TaskbarProgressState taskbarState) + => taskbarInstance.SetProgressState(consoleWindowHandle, taskbarState); + + /// + /// Sets the progress value out of 100. + /// + internal void SetValue(uint progressValue) + => taskbarInstance.SetProgressValue(consoleWindowHandle, progressValue, 100); + } + + private sealed class Terminal + { + [Flags] + private enum ConsoleModes : uint + { + ENABLE_PROCESSED_INPUT = 0x0001, + ENABLE_LINE_INPUT = 0x0002, + ENABLE_ECHO_INPUT = 0x0004, + ENABLE_WINDOW_INPUT = 0x0008, + ENABLE_MOUSE_INPUT = 0x0010, + ENABLE_INSERT_MODE = 0x0020, + ENABLE_QUICK_EDIT_MODE = 0x0040, + ENABLE_EXTENDED_FLAGS = 0x0080, + ENABLE_AUTO_POSITION = 0x0100, + + ENABLE_PROCESSED_OUTPUT = 0x0001, + ENABLE_WRAP_AT_EOL_OUTPUT = 0x0002, + ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004, + DISABLE_NEWLINE_AUTO_RETURN = 0x0008, + ENABLE_LVB_GRID_WORLDWIDE = 0x0010 + } + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool GetConsoleMode(IntPtr hConsoleHandle, out ConsoleModes lpMode); + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool SetConsoleMode(IntPtr hConsoleHandle, ConsoleModes dwMode); + [DllImport("kernel32.dll", SetLastError = true)] + private static extern IntPtr GetStdHandle(int nStdHandle); + private const int STD_OUTPUT_HANDLE = -11; + + private readonly IntPtr consoleHandle; + private uint currentProgress; + + private Terminal(IntPtr handle) + => consoleHandle = handle; + + internal static Terminal? MaybeCreateInstanceAndSetInitialState(TaskbarProgressState initialTaskbarState) + { + IntPtr handle = GetStdHandle(STD_OUTPUT_HANDLE); + if (handle == IntPtr.Zero) + { + return null; + } + if (!GetConsoleMode(handle, out ConsoleModes previousConsoleMode) + || !SetConsoleMode(handle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT)) + { + // If we failed to set virtual terminal processing mode, it is likely due to an older Windows version that does not support it, + // or legacy console. In either case the TaskbarProgressCom will take care of the progress. See https://stackoverflow.com/a/44574463/5703407. + // If we try to write without VT mode, the sequence will be printed for the user to see, which clutters the output. + return null; + } + var terminal = new Terminal(handle); + terminal.WriteStateSequence(initialTaskbarState); + SetConsoleMode(handle, previousConsoleMode); + return terminal; + } + + internal void SetState(TaskbarProgressState taskbarState) + { + GetConsoleMode(consoleHandle, out ConsoleModes previousConsoleMode); + SetConsoleMode(consoleHandle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT); + WriteStateSequence(taskbarState); + SetConsoleMode(consoleHandle, previousConsoleMode); + } + + private void WriteStateSequence(TaskbarProgressState taskbarState) + { + // Write progress state to console for Windows Terminal (https://github.com/microsoft/terminal/issues/6700). + switch (taskbarState) + { + case TaskbarProgressState.NoProgress: + currentProgress = 100; + Console.Write("\x1b]9;4;0;0\x1b\\"); + break; + case TaskbarProgressState.Indeterminate: + currentProgress = 100; + Console.Write("\x1b]9;4;3;0\x1b\\"); + break; + case TaskbarProgressState.Normal: + // Normal state is set when progress is set. + WriteProgressSequence(currentProgress); + break; + case TaskbarProgressState.Error: + Console.Write($"\x1b]9;4;2;{currentProgress}\x1b\\"); + break; + case TaskbarProgressState.Warning: + Console.Write($"\x1b]9;4;4;{currentProgress}\x1b\\"); + break; + } + } + + /// + /// Sets the progress value out of 100. + /// + internal void SetValue(uint progressValue) + { + currentProgress = progressValue; + // Write progress sequence to console for Windows Terminal (https://github.com/microsoft/terminal/discussions/14268). + GetConsoleMode(consoleHandle, out ConsoleModes previousConsoleMode); + SetConsoleMode(consoleHandle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT); + WriteProgressSequence(progressValue); + SetConsoleMode(consoleHandle, previousConsoleMode); + } + + private static void WriteProgressSequence(uint progressValue) + => Console.Write($"\x1b]9;4;1;{progressValue}\x1b\\"); + } + } +} diff --git a/src/BenchmarkDotNet/Helpers/UnitHelper.cs b/src/BenchmarkDotNet/Helpers/UnitHelper.cs new file mode 100644 index 0000000000..9575401776 --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/UnitHelper.cs @@ -0,0 +1,11 @@ +using Perfolizer.Horology; +using Perfolizer.Metrology; + +namespace BenchmarkDotNet.Helpers; + +public static class UnitHelper +{ + public static readonly UnitPresentation DefaultPresentation = new(true, 0, gap: true); + + public static string ToDefaultString(this TimeInterval timeInterval, string? format = null) => timeInterval.ToString(format, null, DefaultPresentation); +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/UserInteractionHelper.cs b/src/BenchmarkDotNet/Helpers/UserInteractionHelper.cs index 7951d168dd..73e83d6ada 100644 --- a/src/BenchmarkDotNet/Helpers/UserInteractionHelper.cs +++ b/src/BenchmarkDotNet/Helpers/UserInteractionHelper.cs @@ -1,3 +1,4 @@ +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Helpers @@ -19,7 +20,7 @@ internal static class UserInteractionHelper /// public static string EscapeCommandExample(string input) { - return !RuntimeInformation.IsWindows() && input.IndexOf('*') >= 0 ? $"'{input}'" : input; + return !OsDetector.IsWindows() && input.IndexOf('*') >= 0 ? $"'{input}'" : input; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/XUnitHelper.cs b/src/BenchmarkDotNet/Helpers/XUnitHelper.cs index 559113a8f4..7f7415bdfa 100644 --- a/src/BenchmarkDotNet/Helpers/XUnitHelper.cs +++ b/src/BenchmarkDotNet/Helpers/XUnitHelper.cs @@ -6,5 +6,5 @@ namespace BenchmarkDotNet.Helpers; internal static class XUnitHelper { public static Lazy IsIntegrationTest = - new (() => AppDomain.CurrentDomain.GetAssemblies().Any(assembly => assembly.GetName().Name == "BenchmarkDotNet.IntegrationTests")); + new(() => AppDomain.CurrentDomain.GetAssemblies().Any(assembly => assembly.GetName().Name == "BenchmarkDotNet.IntegrationTests")); } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs b/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs index 8fe04e5764..0d5113db48 100644 --- a/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs +++ b/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Models; using BenchmarkDotNet.Portability; using JetBrains.Annotations; @@ -14,7 +16,10 @@ public sealed class EnvironmentMode : JobMode public static readonly Characteristic RuntimeCharacteristic = CreateCharacteristic(nameof(Runtime)); public static readonly Characteristic AffinityCharacteristic = CreateCharacteristic(nameof(Affinity)); public static readonly Characteristic GcCharacteristic = CreateCharacteristic(nameof(Gc)); - public static readonly Characteristic> EnvironmentVariablesCharacteristic = CreateCharacteristic>(nameof(EnvironmentVariables)); + + public static readonly Characteristic> EnvironmentVariablesCharacteristic = + CreateCharacteristic>(nameof(EnvironmentVariables)); + public static readonly Characteristic PowerPlanModeCharacteristic = CreateCharacteristic(nameof(PowerPlanMode)); public static readonly Characteristic LargeAddressAwareCharacteristic = CreateCharacteristic(nameof(LargeAddressAware)); @@ -60,7 +65,7 @@ public Jit Jit /// /// Runtime /// - public Runtime Runtime + public Runtime? Runtime { get { return RuntimeCharacteristic[this]; } set { RuntimeCharacteristic[this] = value; } @@ -107,7 +112,7 @@ public bool LargeAddressAware get => LargeAddressAwareCharacteristic[this]; set { - if (value && !RuntimeInformation.IsWindows()) + if (value && !OsDetector.IsWindows()) { throw new NotSupportedException("LargeAddressAware is a Windows-specific concept."); } @@ -133,5 +138,12 @@ public void SetEnvironmentVariable(EnvironmentVariable variable) } internal Runtime GetRuntime() => HasValue(RuntimeCharacteristic) ? Runtime : RuntimeInformation.GetCurrentRuntime(); + + internal BdnEnvironment ToPerfonar() => new() + { + Jit = HasValue(JitCharacteristic) ? Jit : null, + Runtime = HasValue(RuntimeCharacteristic) ? Runtime?.RuntimeMoniker : null, + Affinity = HasValue(AffinityCharacteristic) ? (int)Affinity : null + }; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs b/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs index 5e5f8b5369..277627d2e9 100644 --- a/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs +++ b/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs @@ -1,4 +1,6 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.ComponentModel; using System.Diagnostics.CodeAnalysis; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Engines; @@ -18,6 +20,9 @@ public sealed class InfrastructureMode : JobMode public static readonly Characteristic EngineFactoryCharacteristic = CreateCharacteristic(nameof(EngineFactory)); public static readonly Characteristic BuildConfigurationCharacteristic = CreateCharacteristic(nameof(BuildConfiguration)); public static readonly Characteristic> ArgumentsCharacteristic = CreateCharacteristic>(nameof(Arguments)); + + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This will soon be removed")] public static readonly Characteristic> NuGetReferencesCharacteristic = CreateCharacteristic>(nameof(NuGetReferences)); public static readonly InfrastructureMode InProcess = new InfrastructureMode(InProcessEmitToolchain.Instance); @@ -64,6 +69,8 @@ public IReadOnlyList Arguments set => ArgumentsCharacteristic[this] = value; } + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This will soon be removed")] public IReadOnlyCollection NuGetReferences { get => NuGetReferencesCharacteristic[this]; diff --git a/src/BenchmarkDotNet/Jobs/JobComparer.cs b/src/BenchmarkDotNet/Jobs/JobComparer.cs index 0a0c287523..ac1539ce7c 100644 --- a/src/BenchmarkDotNet/Jobs/JobComparer.cs +++ b/src/BenchmarkDotNet/Jobs/JobComparer.cs @@ -1,12 +1,23 @@ -using System; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Order; +using System; using System.Collections.Generic; -using BenchmarkDotNet.Characteristics; namespace BenchmarkDotNet.Jobs { internal class JobComparer : IComparer, IEqualityComparer { - public static readonly JobComparer Instance = new JobComparer(); + private readonly IComparer Comparer; + + public static readonly JobComparer Instance = new JobComparer(JobOrderPolicy.Default); + public static readonly JobComparer Numeric = new JobComparer(JobOrderPolicy.Numeric); + + public JobComparer(JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default) + { + Comparer = jobOrderPolicy == JobOrderPolicy.Default + ? StringComparer.Ordinal + : new NumericStringComparer(); // TODO: Use `StringComparer.Create(CultureInfo.InvariantCulture, CompareOptions.NumericOrdering)` for .NET10 or greater. + } public int Compare(Job x, Job y) { @@ -39,7 +50,7 @@ public int Compare(Job x, Job y) continue; } - int compare = string.CompareOrdinal( + int compare = Comparer.Compare( presenter.ToPresentation(x, characteristic), presenter.ToPresentation(y, characteristic)); if (compare != 0) @@ -52,5 +63,79 @@ public int Compare(Job x, Job y) public bool Equals(Job x, Job y) => Compare(x, y) == 0; public int GetHashCode(Job obj) => obj.Id.GetHashCode(); + + internal class NumericStringComparer : IComparer + { + public int Compare(string? x, string? y) + { + if (ReferenceEquals(x, y)) return 0; + if (x == null) return -1; + if (y == null) return 1; + + ReadOnlySpan spanX = x.AsSpan(); + ReadOnlySpan spanY = y.AsSpan(); + + int i = 0, j = 0; + + while (i < spanX.Length && j < spanY.Length) + { + char cx = spanX[i]; + char cy = spanY[j]; + + if (!char.IsDigit(cx) || !char.IsDigit(cy)) + { + int cmp = cx.CompareTo(cy); + if (cmp != 0) + return cmp; + + i++; + j++; + continue; + } + + int ixStart = i; + int iyStart = j; + + // Skip leading zeros + while (ixStart < spanX.Length && spanX[ixStart] == '0') ixStart++; + while (iyStart < spanY.Length && spanY[iyStart] == '0') iyStart++; + + int ix = ixStart; + int iy = iyStart; + + // Skip digits + while (ix < spanX.Length && char.IsDigit(spanX[ix])) ix++; + while (iy < spanY.Length && char.IsDigit(spanY[iy])) iy++; + + int lenX = ix - ixStart; + int lenY = iy - iyStart; + + // Compare by digits length + if (lenX != lenY) + return lenX.CompareTo(lenY); + + // Compare digits + for (int k = 0; k < lenX; k++) + { + int cmp = spanX[ixStart + k].CompareTo(spanY[iyStart + k]); + if (cmp != 0) + return cmp; + } + + // Compare by leading zeros + int leadingZerosX = ixStart - i; + int leadingZerosY = iyStart - j; + if (leadingZerosX != leadingZerosY) + return 0; // Leading zero differences are ignored (`CompareOptions.NumericOrdering` behavior of .NET) + + // Move to the next character after the digits + i = ix; + j = iy; + } + + // Compare remaining chars + return (spanX.Length - i).CompareTo(spanY.Length - j); + } + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index 2a236dc44f..0424528ad4 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -339,6 +339,8 @@ public static Job WithEnvironmentVariable(this Job job, string key, string value /// (optional)Indicate the URI of the NuGet package source to use during the restore operation. /// (optional)Allows prerelease packages to be installed. /// + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This method will soon be removed, please start using .WithMsBuildArguments() instead.")] public static Job WithNuGet(this Job job, string packageName, string? packageVersion = null, Uri? source = null, bool prerelease = false) => job.WithCore(j => j.Infrastructure.NuGetReferences = new NuGetReferenceList(j.Infrastructure.NuGetReferences ?? Array.Empty()) @@ -352,9 +354,14 @@ public static Job WithNuGet(this Job job, string packageName, string? packageVer /// /// A collection of NuGet dependencies /// + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This method will soon be removed, please start using .WithMsBuildArguments() instead.")] public static Job WithNuGet(this Job job, NuGetReferenceList nuGetReferences) => job.WithCore(j => j.Infrastructure.NuGetReferences = nuGetReferences); + public static Job WithMsBuildArguments(this Job job, params string[] msBuildArguments) + => job.WithArguments([.. msBuildArguments.Select(a => new MsBuildArgument(a))]); + // Accuracy /// /// Maximum acceptable error for a benchmark (by default, BenchmarkDotNet continue iterations until the actual error is less than the specified error). @@ -436,5 +443,12 @@ private static Job WithCore(this Job job, Action updateCallback) updateCallback(newJob); return newJob; } + + internal static bool HasDynamicBuildCharacteristic(this Job job) +#pragma warning disable CS0618 // Type or member is obsolete + => job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic) +#pragma warning restore CS0618 // Type or member is obsolete + || job.HasValue(InfrastructureMode.BuildConfigurationCharacteristic) + || job.HasValue(InfrastructureMode.ArgumentsCharacteristic); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/JobIdGenerator.cs b/src/BenchmarkDotNet/Jobs/JobIdGenerator.cs index fa8796d112..a208b3c9b7 100644 --- a/src/BenchmarkDotNet/Jobs/JobIdGenerator.cs +++ b/src/BenchmarkDotNet/Jobs/JobIdGenerator.cs @@ -10,12 +10,23 @@ public static string GenerateRandomId(Job job) string presentation = CharacteristicSetPresenter.Display.ToPresentation(job); if (presentation == "") return "DefaultJob"; - int seed = presentation.GetHashCode(); + int seed = GetStableHashCode(presentation); var random = new Random(seed); string id = ""; for (int i = 0; i < 6; i++) - id += (char) ('A' + random.Next(26)); + id += (char)('A' + random.Next(26)); return "Job-" + id; } + + // Compute string hash value with DJB2 algorithm. + private static int GetStableHashCode(string value) + { + uint hash = 5381; + foreach (char c in value) + { + hash = ((hash << 5) + hash) + c; // hash * 32 + hash + c + } + return unchecked((int)hash); + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/NugetReference.cs b/src/BenchmarkDotNet/Jobs/NugetReference.cs index 0db3a028de..89dc4c3734 100644 --- a/src/BenchmarkDotNet/Jobs/NugetReference.cs +++ b/src/BenchmarkDotNet/Jobs/NugetReference.cs @@ -1,8 +1,11 @@ using System; +using System.ComponentModel; using System.Text.RegularExpressions; namespace BenchmarkDotNet.Jobs { + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This type will soon be removed")] public class NuGetReference : IEquatable { public NuGetReference(string packageName, string packageVersion, Uri? source = null, bool prerelease = false) diff --git a/src/BenchmarkDotNet/Jobs/NugetReferenceList.cs b/src/BenchmarkDotNet/Jobs/NugetReferenceList.cs index 13c727c28f..025353b24f 100644 --- a/src/BenchmarkDotNet/Jobs/NugetReferenceList.cs +++ b/src/BenchmarkDotNet/Jobs/NugetReferenceList.cs @@ -1,12 +1,15 @@ using System; using System.Collections; using System.Collections.Generic; +using System.ComponentModel; namespace BenchmarkDotNet.Jobs { /// /// An ordered list of NuGet references. Does not allow duplicate references with the same PackageName. /// + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This type will soon be removed")] public class NuGetReferenceList : IReadOnlyCollection { private readonly List references = new List(); diff --git a/src/BenchmarkDotNet/Jobs/RunMode.cs b/src/BenchmarkDotNet/Jobs/RunMode.cs index a76b5dc832..3e1e56c22f 100644 --- a/src/BenchmarkDotNet/Jobs/RunMode.cs +++ b/src/BenchmarkDotNet/Jobs/RunMode.cs @@ -2,6 +2,8 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Models; using Perfolizer.Horology; namespace BenchmarkDotNet.Jobs @@ -9,7 +11,8 @@ namespace BenchmarkDotNet.Jobs [SuppressMessage("ReSharper", "UnusedMember.Global")] public sealed class RunMode : JobMode { - public static readonly Characteristic RunStrategyCharacteristic = Characteristic.Create(nameof(RunStrategy), RunStrategy.Throughput); + public static readonly Characteristic RunStrategyCharacteristic = + Characteristic.Create(nameof(RunStrategy), RunStrategy.Throughput); public static readonly Characteristic LaunchCountCharacteristic = CreateCharacteristic(nameof(LaunchCount)); public static readonly Characteristic InvocationCountCharacteristic = CreateCharacteristic(nameof(InvocationCount)); @@ -61,13 +64,9 @@ public sealed class RunMode : JobMode }.Freeze(); - public RunMode() : this(null) - { - } + public RunMode() : this(null) { } - private RunMode(string id) : base(id) - { - } + private RunMode(string id) : base(id) { } /// /// Available values: Throughput and ColdStart. @@ -191,5 +190,21 @@ public bool MemoryRandomization get => MemoryRandomizationCharacteristic[this]; set => MemoryRandomizationCharacteristic[this] = value; } + + internal BdnExecution ToPerfonar() => new() + { + LaunchCount = HasValue(LaunchCountCharacteristic) ? LaunchCount : null, + WarmupCount = HasValue(WarmupCountCharacteristic) ? WarmupCount : null, + IterationCount = HasValue(IterationCountCharacteristic) ? IterationCount : null, + IterationTimeMs = HasValue(IterationTimeCharacteristic) ? IterationTime.ToMilliseconds().RoundToLong() : null, + InvocationCount = HasValue(InvocationCountCharacteristic) ? InvocationCount : null, + UnrollFactor = HasValue(UnrollFactorCharacteristic) ? UnrollFactor : null, + MinIterationCount = HasValue(MinIterationCountCharacteristic) ? MinIterationCount : null, + MaxIterationCount = HasValue(MaxIterationCountCharacteristic) ? MaxIterationCount : null, + MinWarmupIterationCount = HasValue(MinWarmupIterationCountCharacteristic) ? MinWarmupIterationCount : null, + MaxWarmupIterationCount = HasValue(MaxWarmupIterationCountCharacteristic) ? MaxWarmupIterationCount : null, + MemoryRandomization = HasValue(MemoryRandomizationCharacteristic) ? MemoryRandomization : null, + RunStrategy = HasValue(RunStrategyCharacteristic) ? RunStrategy : null, + }; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Loggers/Broker.cs b/src/BenchmarkDotNet/Loggers/Broker.cs index cec11a3091..277cd3c430 100644 --- a/src/BenchmarkDotNet/Loggers/Broker.cs +++ b/src/BenchmarkDotNet/Loggers/Broker.cs @@ -15,9 +15,7 @@ internal class Broker { private readonly ILogger logger; private readonly Process process; - private readonly IDiagnoser diagnoser; private readonly AnonymousPipeServerStream inputFromBenchmark, acknowledgments; - private readonly DiagnoserActionParameters diagnoserActionParameters; private readonly ManualResetEvent finished; public Broker(ILogger logger, Process process, IDiagnoser diagnoser, @@ -25,10 +23,10 @@ public Broker(ILogger logger, Process process, IDiagnoser diagnoser, { this.logger = logger; this.process = process; - this.diagnoser = diagnoser; + this.Diagnoser = diagnoser; this.inputFromBenchmark = inputFromBenchmark; this.acknowledgments = acknowledgments; - diagnoserActionParameters = new DiagnoserActionParameters(process, benchmarkCase, benchmarkId); + DiagnoserActionParameters = new DiagnoserActionParameters(process, benchmarkCase, benchmarkId); finished = new ManualResetEvent(false); Results = new List(); @@ -38,6 +36,10 @@ public Broker(ILogger logger, Process process, IDiagnoser diagnoser, process.Exited += OnProcessExited; } + internal IDiagnoser Diagnoser { get; } + + internal DiagnoserActionParameters DiagnoserActionParameters { get; } + internal List Results { get; } internal List PrefixedOutput { get; } @@ -74,8 +76,8 @@ private void OnProcessExited(object sender, EventArgs e) private void ProcessDataBlocking() { - using StreamReader reader = new (inputFromBenchmark, AnonymousPipesHost.UTF8NoBOM, detectEncodingFromByteOrderMarks: false); - using StreamWriter writer = new (acknowledgments, AnonymousPipesHost.UTF8NoBOM, bufferSize: 1); + using StreamReader reader = new(inputFromBenchmark, AnonymousPipesHost.UTF8NoBOM, detectEncodingFromByteOrderMarks: false); + using StreamWriter writer = new(acknowledgments, AnonymousPipesHost.UTF8NoBOM, bufferSize: 1); // Flush the data to the Stream after each write, otherwise the client will wait for input endlessly! writer.AutoFlush = true; @@ -90,7 +92,7 @@ private void ProcessDataBlocking() } else if (Engine.Signals.TryGetSignal(line, out var signal)) { - diagnoser?.Handle(signal, diagnoserActionParameters); + Diagnoser?.Handle(signal, DiagnoserActionParameters); writer.WriteLine(Engine.Signals.Acknowledgment); diff --git a/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs b/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs index a1c7795d53..d201f5da75 100644 --- a/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs +++ b/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Portability; using JetBrains.Annotations; @@ -15,7 +16,7 @@ public sealed class ConsoleLogger : ILogger public static readonly ILogger Ascii = new ConsoleLogger(false); public static readonly ILogger Unicode = new ConsoleLogger(true); private static readonly bool ConsoleSupportsColors - = !(RuntimeInformation.IsAndroid() || RuntimeInformation.IsIOS() || RuntimeInformation.IsWasm || RuntimeInformation.IsTvOS()); + = !(OsDetector.IsAndroid() || OsDetector.IsIOS() || RuntimeInformation.IsWasm || OsDetector.IsTvOS()); private readonly bool unicodeSupport; private readonly Dictionary colorScheme; @@ -79,6 +80,7 @@ private static Dictionary CreateColorfulScheme() => { LogKind.Statistic, ConsoleColor.Cyan }, { LogKind.Info, ConsoleColor.DarkYellow }, { LogKind.Error, ConsoleColor.Red }, + { LogKind.Warning, ConsoleColor.Yellow }, { LogKind.Hint, ConsoleColor.DarkCyan } }; diff --git a/src/BenchmarkDotNet/Loggers/LinqPadLogger.cs b/src/BenchmarkDotNet/Loggers/LinqPadLogger.cs index 01fa682c99..cbb023848f 100644 --- a/src/BenchmarkDotNet/Loggers/LinqPadLogger.cs +++ b/src/BenchmarkDotNet/Loggers/LinqPadLogger.cs @@ -82,6 +82,7 @@ private static IReadOnlyDictionary CreateDarkScheme() => { LogKind.Statistic, "#00FFFF" }, { LogKind.Info, "#808000" }, { LogKind.Error, "#FF0000" }, + { LogKind.Warning, "#FFFF00" }, { LogKind.Hint, "#008080" } }; @@ -95,6 +96,7 @@ private static IReadOnlyDictionary CreateLightScheme() => { LogKind.Statistic, "#008080" }, { LogKind.Info, "#808000" }, { LogKind.Error, "#FF0000" }, + { LogKind.Warning, "#FFFF00" }, { LogKind.Hint, "#008080" } }; } diff --git a/src/BenchmarkDotNet/Loggers/LogKind.cs b/src/BenchmarkDotNet/Loggers/LogKind.cs index 8f3dff72a8..8fa4660aa8 100644 --- a/src/BenchmarkDotNet/Loggers/LogKind.cs +++ b/src/BenchmarkDotNet/Loggers/LogKind.cs @@ -2,6 +2,6 @@ { public enum LogKind { - Default, Help, Header, Result, Statistic, Info, Error, Hint + Default, Help, Header, Result, Statistic, Info, Error, Hint, Warning } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Loggers/LoggerExtensions.cs b/src/BenchmarkDotNet/Loggers/LoggerExtensions.cs index 866620c3fb..1d75341bb8 100644 --- a/src/BenchmarkDotNet/Loggers/LoggerExtensions.cs +++ b/src/BenchmarkDotNet/Loggers/LoggerExtensions.cs @@ -18,6 +18,8 @@ public static class LoggerExtensions public static void WriteLineError(this ILogger logger, string text) => logger.WriteLine(LogKind.Error, text); + public static void WriteLineWarning(this ILogger logger, string text) => logger.WriteLine(LogKind.Warning, text); + public static void WriteLineHint(this ILogger logger, string text) => logger.WriteLine(LogKind.Hint, text); public static void Write(this ILogger logger, string text) => logger.Write(LogKind.Default, text); @@ -36,6 +38,8 @@ public static class LoggerExtensions public static void WriteError(this ILogger logger, string text) => logger.Write(LogKind.Error, text); + public static void WriteWarning(this ILogger logger, string text) => logger.Write(LogKind.Warning, text); + [PublicAPI] public static void WriteHint(this ILogger logger, string text) => logger.Write(LogKind.Hint, text); } diff --git a/src/BenchmarkDotNet/Mathematics/LegacyConfidenceInterval.cs b/src/BenchmarkDotNet/Mathematics/LegacyConfidenceInterval.cs new file mode 100644 index 0000000000..f1fa247f21 --- /dev/null +++ b/src/BenchmarkDotNet/Mathematics/LegacyConfidenceInterval.cs @@ -0,0 +1,167 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using BenchmarkDotNet.Helpers; +using JetBrains.Annotations; +using Perfolizer.Mathematics.Common; + +namespace BenchmarkDotNet.Mathematics; + +public enum LegacyConfidenceLevel +{ + /// + /// 50.0% confidence interval + /// + [PublicAPI] L50, + + /// + /// 70.0% confidence interval + /// + [PublicAPI] L70, + + /// + /// 75.0% confidence interval + /// + [PublicAPI] L75, + + /// + /// 80.0% confidence interval + /// + [PublicAPI] L80, + + /// + /// 85.0% confidence interval + /// + [PublicAPI] L85, + + /// + /// 90.0% confidence interval + /// + [PublicAPI] L90, + + /// + /// 92.0% confidence interval + /// + [PublicAPI] L92, + + /// + /// 95.0% confidence interval + /// + [PublicAPI] L95, + + /// + /// 96.0% confidence interval + /// + [PublicAPI] L96, + + /// + /// 97.0% confidence interval + /// + [PublicAPI] L97, + + /// + /// 98.0% confidence interval + /// + [PublicAPI] L98, + + /// + /// 99.0% confidence interval + /// + [PublicAPI] L99, + + /// + /// 99.9% confidence interval + /// + [PublicAPI] L999 +} + +public static class ConfidenceLevelExtensions +{ + private static readonly Dictionary ConfidenceLevelDetails = CreateConfidenceLevelMapping(); + + /// + /// Calculates Z value (z-star) for confidence interval + /// + /// ConfidenceLevel for a confidence interval + /// Sample size (n >= 3) + public static double GetZValue(this LegacyConfidenceLevel level, int n) + { + return new ConfidenceIntervalEstimator(n, 0, 1).ZLevel(new ConfidenceLevel(level.ToPercent())); + } + + public static string ToPercentStr(this LegacyConfidenceLevel level) + { + string s = level.ToString().Substring(1); + if (s.Length > 2) + s = s.Substring(0, 2) + "." + s.Substring(2); + return s + "%"; + } + + [PublicAPI] public static double ToPercent(this LegacyConfidenceLevel level) + { + (int value, int digits) = ConfidenceLevelDetails[level]; + + return value / Math.Pow(10, digits); + } + + private static Dictionary CreateConfidenceLevelMapping() + => Enum.GetValues(typeof(LegacyConfidenceLevel)) + .Cast() + .ToDictionary( + confidenceLevel => confidenceLevel, + confidenceLevel => + { + string textRepresentation = confidenceLevel.ToString().Substring(1); + + return (int.Parse(textRepresentation), textRepresentation.Length); + }); +} + +public struct LegacyConfidenceInterval +{ + [PublicAPI] public int N { get; } + [PublicAPI] public double Mean { get; } + [PublicAPI] public double StandardError { get; } + + [PublicAPI] public LegacyConfidenceLevel Level { get; } + [PublicAPI] public double Margin { get; } + + [PublicAPI] public double Lower { get; } + [PublicAPI] public double Upper { get; } + + public LegacyConfidenceInterval(double mean, double standardError, int n, LegacyConfidenceLevel level = LegacyConfidenceLevel.L999) + { + N = n; + Mean = mean; + StandardError = standardError; + Level = level; + Margin = n <= 2 ? double.NaN : standardError * level.GetZValue(n); + Lower = mean - Margin; + Upper = mean + Margin; + } + + public bool Contains(double value) => Lower - 1e-9 < value && value < Upper + 1e-9; + + private string GetLevelHint(bool showLevel = true) => showLevel ? $" (CI {Level.ToPercentStr()})" : ""; + + public override string ToString() => ToString(DefaultCultureInfo.Instance); + + public string ToString(CultureInfo cultureInfo, string format = "0.##", bool showLevel = true) + { + return ToString(x => x.ToString(format, cultureInfo), showLevel); + } + + public string ToString(Func formatter, bool showLevel = true) + { + var builder = new StringBuilder(); + builder.Append('['); + builder.Append(formatter(Lower)); + builder.Append("; "); + builder.Append(formatter(Upper)); + builder.Append("]"); + builder.Append(GetLevelHint(showLevel)); + return builder.ToString(); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Mathematics/MathHelper.cs b/src/BenchmarkDotNet/Mathematics/MathHelper.cs index 01f5491763..d6250f1618 100644 --- a/src/BenchmarkDotNet/Mathematics/MathHelper.cs +++ b/src/BenchmarkDotNet/Mathematics/MathHelper.cs @@ -1,10 +1,13 @@ -using static System.Math; +using Perfolizer.Mathematics.Common; +using Perfolizer.Metrology; +using static System.Math; namespace BenchmarkDotNet.Mathematics { - public static class MathHelper + internal static class MathHelper { - + public static readonly Threshold DefaultThreshold = PercentValue.Of(2).ToThreshold(); + public static readonly SignificanceLevel DefaultSignificanceLevel = SignificanceLevel.P1E5; public static int Clamp(int value, int min, int max) => Min(Max(value, min), max); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Mathematics/MeasurementsStatistics.cs b/src/BenchmarkDotNet/Mathematics/MeasurementsStatistics.cs index 0ec336b101..41d76fe849 100644 --- a/src/BenchmarkDotNet/Mathematics/MeasurementsStatistics.cs +++ b/src/BenchmarkDotNet/Mathematics/MeasurementsStatistics.cs @@ -26,13 +26,13 @@ internal readonly ref struct MeasurementsStatistics /// /// 99.9% confidence interval in nanoseconds. /// - public ConfidenceInterval ConfidenceInterval { get; } + public ConfidenceInterval LegacyConfidenceInterval { get; } - private MeasurementsStatistics(double standardError, double mean, ConfidenceInterval confidenceInterval) + private MeasurementsStatistics(double standardError, double mean, ConfidenceInterval legacyConfidenceInterval) { StandardError = standardError; Mean = mean; - ConfidenceInterval = confidenceInterval; + LegacyConfidenceInterval = legacyConfidenceInterval; } public static MeasurementsStatistics Calculate(List measurements, OutlierMode outlierMode) @@ -47,7 +47,8 @@ public static MeasurementsStatistics Calculate(List measurements, O double variance = Variance(measurements, n, mean); double standardDeviation = Math.Sqrt(variance); double standardError = standardDeviation / Math.Sqrt(n); - var confidenceInterval = new ConfidenceInterval(mean, standardError, n); + var confidenceIntervalEstimator = new ConfidenceIntervalEstimator(n, mean, standardError); + var confidenceInterval = confidenceIntervalEstimator.ConfidenceInterval(ConfidenceLevel.L999); if (outlierMode == OutlierMode.DontRemove) // most simple scenario is done without allocations! but this is not the default case return new MeasurementsStatistics(standardError, mean, confidenceInterval); @@ -74,7 +75,8 @@ public static MeasurementsStatistics Calculate(List measurements, O variance = VarianceWithoutOutliers(outlierMode, measurements, n, mean, lowerFence, upperFence); standardDeviation = Math.Sqrt(variance); standardError = standardDeviation / Math.Sqrt(n); - confidenceInterval = new ConfidenceInterval(mean, standardError, n); + confidenceIntervalEstimator = new ConfidenceIntervalEstimator(n, mean, standardError); + confidenceInterval = confidenceIntervalEstimator.ConfidenceInterval(ConfidenceLevel.L999); return new MeasurementsStatistics(standardError, mean, confidenceInterval); } @@ -113,7 +115,8 @@ private static double Variance(List measurements, int n, double mea return variance; } - private static double VarianceWithoutOutliers(OutlierMode outlierMode, List measurements, int n, double mean, double lowerFence, double upperFence) + private static double VarianceWithoutOutliers(OutlierMode outlierMode, List measurements, int n, double mean, double lowerFence, + double upperFence) { if (n == 1) return 0; @@ -151,4 +154,4 @@ private static bool IsOutlier(OutlierMode outlierMode, double value, double lowe } } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Mathematics/PercentileValues.cs b/src/BenchmarkDotNet/Mathematics/PercentileValues.cs index 34abe47430..7241206517 100644 --- a/src/BenchmarkDotNet/Mathematics/PercentileValues.cs +++ b/src/BenchmarkDotNet/Mathematics/PercentileValues.cs @@ -4,42 +4,14 @@ using System.Text; using BenchmarkDotNet.Helpers; using JetBrains.Annotations; +using Perfolizer; using Perfolizer.Mathematics.QuantileEstimators; namespace BenchmarkDotNet.Mathematics { public class PercentileValues { - /// - /// Calculates the Nth percentile from the set of values - /// - /// - /// The implementation is expected to be consistent with the one from Excel. - /// It's a quite common to export bench output into .csv for further analysis - /// And it's a good idea to have same results from all tools being used. - /// - /// Sequence of the values to be calculated - /// Value in range 0..100 - /// Percentile from the set of values - // Based on: http://stackoverflow.com/a/8137526 - private static double Percentile(IReadOnlyList sortedValues, int percentile) - { - if (sortedValues == null) - throw new ArgumentNullException(nameof(sortedValues)); - if (percentile < 0 || percentile > 100) - { - throw new ArgumentOutOfRangeException( - nameof(percentile), percentile, - "The percentile arg should be in range of 0 - 100."); - } - - if (sortedValues.Count == 0) - return 0; - - return SimpleQuantileEstimator.Instance.GetQuantileFromSorted(sortedValues, percentile / 100.0); - } - - [PublicAPI] public double Percentile(int percentile) => Percentile(SortedValues, percentile); + [PublicAPI] public double Percentile(int percentile) => SimpleQuantileEstimator.Instance.Quantile(new Sample(SortedValues), percentile / 100.0); private IReadOnlyList SortedValues { get; } diff --git a/src/BenchmarkDotNet/Mathematics/RankHelper.cs b/src/BenchmarkDotNet/Mathematics/RankHelper.cs index 8f7587a9ea..526ad46a1d 100644 --- a/src/BenchmarkDotNet/Mathematics/RankHelper.cs +++ b/src/BenchmarkDotNet/Mathematics/RankHelper.cs @@ -1,7 +1,8 @@ using System; using System.Linq; +using Perfolizer.Mathematics.Common; using Perfolizer.Mathematics.SignificanceTesting; -using Perfolizer.Mathematics.Thresholds; +using Perfolizer.Mathematics.SignificanceTesting.MannWhitney; namespace BenchmarkDotNet.Mathematics { @@ -9,10 +10,7 @@ internal static class RankHelper { public static int[] GetRanks(params Statistics[] stats) { - var values = stats. - Select((s, index) => new { Stats = s, Index = index }). - OrderBy(pair => pair.Stats.Mean). - ToArray(); + var values = stats.Select((s, index) => new { Stats = s, Index = index }).OrderBy(pair => pair.Stats.Mean).ToArray(); int n = values.Length; var ranks = new int[n]; @@ -33,14 +31,9 @@ public static int[] GetRanks(params Statistics[] stats) private static bool AreSame(Statistics x, Statistics y) { - if (x.N >= 3 && y.N >= 3) - { - var tost = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, x.SortedValues.ToArray(), y.SortedValues.ToArray(), RelativeThreshold.Default); - if (tost.Conclusion != EquivalenceTestConclusion.Unknown) - return tost.Conclusion == EquivalenceTestConclusion.Same; - } - - return Math.Abs(x.Mean - y.Mean) < Math.Abs(x.Mean + y.Mean) / 2 * 0.01; + var test = new SimpleEquivalenceTest(MannWhitneyTest.Instance); + var comparisonResult = test.Perform(x.Sample, y.Sample, MathHelper.DefaultThreshold, MathHelper.DefaultSignificanceLevel); + return comparisonResult == ComparisonResult.Indistinguishable; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Mathematics/Statistics.cs b/src/BenchmarkDotNet/Mathematics/Statistics.cs index 35743d009c..be6f47db6c 100644 --- a/src/BenchmarkDotNet/Mathematics/Statistics.cs +++ b/src/BenchmarkDotNet/Mathematics/Statistics.cs @@ -3,6 +3,8 @@ using System.Linq; using BenchmarkDotNet.Extensions; using JetBrains.Annotations; +using Perfolizer; +using Perfolizer.Horology; using Perfolizer.Mathematics.Common; using Perfolizer.Mathematics.OutlierDetection; using Perfolizer.Mathematics.QuantileEstimators; @@ -11,8 +13,8 @@ namespace BenchmarkDotNet.Mathematics { public class Statistics { + internal Sample Sample { get; } public IReadOnlyList OriginalValues { get; } - internal IReadOnlyList SortedValues { get; } public int N { get; } public double Min { get; } public double LowerFence { get; } @@ -31,26 +33,28 @@ public class Statistics public double StandardDeviation { get; } public double Skewness { get; } public double Kurtosis { get; } - public ConfidenceInterval ConfidenceInterval { get; } + private ConfidenceIntervalEstimator ConfidenceIntervalEstimator { get; } + internal ConfidenceInterval PerfolizerConfidenceInterval { get; } + public LegacyConfidenceInterval ConfidenceInterval { get; } public PercentileValues Percentiles { get; } - private OutlierDetector outlierDetector; + private readonly TukeyOutlierDetector outlierDetector; public Statistics(params double[] values) : this(values.ToList()) { } public Statistics(IEnumerable values) : - this(values.Select(value => (double) value)) { } + this(values.Select(value => (double)value)) { } - public Statistics(IEnumerable values) + public Statistics(IEnumerable values) : this(new Sample(values.ToArray(), TimeUnit.Nanosecond)) { } + + public Statistics(Sample sample) { - OriginalValues = values.Where(d => !double.IsNaN(d)).ToArray(); - SortedValues = OriginalValues.OrderBy(value => value).ToArray(); - N = SortedValues.Count; - if (N == 0) - throw new InvalidOperationException("Sequence of values contains no elements, Statistics can't be calculated"); + Sample = sample; + OriginalValues = sample.Values; + N = Sample.Size; - var quartiles = Quartiles.FromSorted(SortedValues); + var quartiles = Quartiles.Create(Sample); Min = quartiles.Min; Q1 = quartiles.Q1; Median = quartiles.Median; @@ -58,33 +62,35 @@ public Statistics(IEnumerable values) Max = quartiles.Max; InterquartileRange = quartiles.InterquartileRange; - var moments = Moments.Create(SortedValues); + var moments = Moments.Create(Sample); Mean = moments.Mean; StandardDeviation = moments.StandardDeviation; Variance = moments.Variance; Skewness = moments.Skewness; Kurtosis = moments.Kurtosis; - var tukey = TukeyOutlierDetector.FromQuartiles(quartiles); + var tukey = TukeyOutlierDetector.Create(Sample); LowerFence = tukey.LowerFence; UpperFence = tukey.UpperFence; - AllOutliers = SortedValues.Where(tukey.IsOutlier).ToArray(); - LowerOutliers = SortedValues.Where(tukey.IsLowerOutlier).ToArray(); - UpperOutliers = SortedValues.Where(tukey.IsUpperOutlier).ToArray(); + AllOutliers = Sample.SortedValues.Where(tukey.IsOutlier).ToArray(); + LowerOutliers = Sample.SortedValues.Where(tukey.IsLowerOutlier).ToArray(); + UpperOutliers = Sample.SortedValues.Where(tukey.IsUpperOutlier).ToArray(); outlierDetector = tukey; StandardError = StandardDeviation / Math.Sqrt(N); - ConfidenceInterval = new ConfidenceInterval(Mean, StandardError, N); - Percentiles = new PercentileValues(SortedValues); + ConfidenceIntervalEstimator = new ConfidenceIntervalEstimator(Sample.Size, Mean, StandardError); + PerfolizerConfidenceInterval = ConfidenceIntervalEstimator.ConfidenceInterval(ConfidenceLevel.L999); + ConfidenceInterval = new LegacyConfidenceInterval(PerfolizerConfidenceInterval.Estimation, StandardError, N, LegacyConfidenceLevel.L999); + Percentiles = new PercentileValues(Sample.SortedValues); } - [PublicAPI] public ConfidenceInterval GetConfidenceInterval(ConfidenceLevel level, int n) => new ConfidenceInterval(Mean, StandardError, n, level); + [PublicAPI] public ConfidenceInterval GetConfidenceInterval(ConfidenceLevel level) => ConfidenceIntervalEstimator.ConfidenceInterval(level); [PublicAPI] public bool IsLowerOutlier(double value) => outlierDetector.IsLowerOutlier(value); [PublicAPI] public bool IsUpperOutlier(double value) => outlierDetector.IsUpperOutlier(value); [PublicAPI] public bool IsOutlier(double value) => outlierDetector.IsOutlier(value); - [PublicAPI] public double[] WithoutOutliers() => SortedValues.Where(value => !IsOutlier(value)).ToArray(); + [PublicAPI] public double[] WithoutOutliers() => outlierDetector.WithoutAllOutliers(Sample.Values).ToArray(); - [PublicAPI] public double CalcCentralMoment(int k) => SortedValues.Average(x => (x - Mean).Pow(k)); + [PublicAPI] public double CalcCentralMoment(int k) => Sample.SortedValues.Average(x => (x - Mean).Pow(k)); public bool IsActualOutlier(double value, OutlierMode outlierMode) { @@ -121,18 +127,7 @@ public double[] GetActualOutliers(OutlierMode outlierMode) } } - public override string ToString() - { - switch (N) - { - case 1: - return $"{SortedValues[0]} (N = 1)"; - case 2: - return $"{SortedValues[0]},{SortedValues[1]} (N = 2)"; - default: - return $"{Mean} +- {ConfidenceInterval.Margin} (N = {N})"; - } - } + public override string ToString() => Sample.ToString(); /// /// Returns true, if this statistics can be inverted (see ). @@ -142,7 +137,7 @@ public override string ToString() /// /// Statistics for [1/X]. If Min is less then or equal to 0, returns null. /// - public Statistics Invert() => CanBeInverted() ? new Statistics(SortedValues.Select(x => 1 / x)) : null; + public Statistics Invert() => CanBeInverted() ? new Statistics(Sample.SortedValues.Select(x => 1 / x)) : null; /// /// Mean for [X*Y]. @@ -168,13 +163,15 @@ public static Statistics Divide(Statistics x, Statistics y) throw new ArgumentOutOfRangeException(nameof(x), "Argument doesn't contain any values"); if (y.N < 1) throw new ArgumentOutOfRangeException(nameof(y), "Argument doesn't contain any values"); - int n = Math.Min(x.N, y.N); - var z = new double[n]; - for (int i = 0; i < n; i++) + + double[]? z = new double[x.N * y.N]; + int k = 0; + for (int i = 0; i < x.N; i++) + for (int j = 0; j < y.N; j++) { - if (Math.Abs(y.OriginalValues[i]) < 1e-9) - throw new DivideByZeroException($"y[{i}] is {y.OriginalValues[i]}"); - z[i] = x.OriginalValues[i] / y.OriginalValues[i]; + if (Math.Abs(y.Sample.Values[j]) < 1e-9) + throw new DivideByZeroException($"y[{j}] is {y.Sample.Values[j]}"); + z[k++] = x.Sample.Values[i] / y.Sample.Values[j]; } return new Statistics(z); diff --git a/src/BenchmarkDotNet/Models/BdnBenchmark.cs b/src/BenchmarkDotNet/Models/BdnBenchmark.cs new file mode 100644 index 0000000000..19eb4e9674 --- /dev/null +++ b/src/BenchmarkDotNet/Models/BdnBenchmark.cs @@ -0,0 +1,36 @@ +using System.Text; +using BenchmarkDotNet.Extensions; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Models; + +internal class BdnBenchmark : BenchmarkInfo +{ + public string Namespace { get; set; } = ""; + public string Type { get; set; } = ""; + public string Method { get; set; } = ""; + public string Parameters { get; set; } = ""; + public string? HardwareIntrinsics { get; set; } = ""; + + // TODO: Improve + public override string? GetDisplay() + { + if (Display != null) return Display; + + var builder = new StringBuilder(); + builder.Append($"{Namespace}"); + if (Type.IsNotBlank()) + { + if (builder.Length > 0) + builder.Append('.'); + builder.Append(Type); + } + if (Method.IsNotBlank()) + { + if (builder.Length > 0) + builder.Append('.'); + builder.Append(Method); + } + return builder.ToString(); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Models/BdnEnvironment.cs b/src/BenchmarkDotNet/Models/BdnEnvironment.cs new file mode 100644 index 0000000000..1256d83c6f --- /dev/null +++ b/src/BenchmarkDotNet/Models/BdnEnvironment.cs @@ -0,0 +1,12 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Models; + +internal class BdnEnvironment : EnvironmentInfo +{ + public RuntimeMoniker? Runtime { get; set; } + public Jit? Jit { get; set; } + public int? Affinity { get; set; } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Models/BdnExecution.cs b/src/BenchmarkDotNet/Models/BdnExecution.cs new file mode 100644 index 0000000000..0031c3897e --- /dev/null +++ b/src/BenchmarkDotNet/Models/BdnExecution.cs @@ -0,0 +1,83 @@ +using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Engines; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Models; + +internal class BdnExecution : ExecutionInfo +{ + /// + /// Available values: Throughput and ColdStart. + /// Throughput: default strategy which allows to get good precision level. + /// ColdStart: should be used only for measuring cold start of the application or testing purpose. + /// Monitoring: no overhead evaluating, with several target iterations. Perfect for macrobenchmarks without a steady state with high variance. + /// + public RunStrategy? RunStrategy { get; set; } + + /// + /// How many times we should launch process with target benchmark. + /// + public int? LaunchCount { get; set; } + + /// + /// How many warmup iterations should be performed. + /// + public int? WarmupCount { get; set; } + + /// + /// How many target iterations should be performed + /// If specified, will be ignored. + /// If specified, will be ignored. + /// + public int? IterationCount { get; set; } + + /// + /// Desired time of execution of an iteration. Used by Pilot stage to estimate the number of invocations per iteration. + /// The default value is 500 milliseconds. + /// + public long? IterationTimeMs { get; set; } + + /// + /// Invocation count in a single iteration. + /// If specified, will be ignored. + /// If specified, it must be a multiple of . + /// + public long? InvocationCount { get; set; } + + /// + /// How many times the benchmark method will be invoked per one iteration of a generated loop. + /// + public int? UnrollFactor { get; set; } + + /// + /// Minimum count of target iterations that should be performed + /// The default value is 15 + /// If you set this value to below 15, then is not going to work + /// + public int? MinIterationCount { get; set; } + + /// + /// Maximum count of target iterations that should be performed + /// The default value is 100 + /// If you set this value to below 15, then is not going to work + /// + public int? MaxIterationCount { get; set; } + + /// + /// Minimum count of warmup iterations that should be performed + /// The default value is 6 + /// + public int? MinWarmupIterationCount { get; set; } + + /// + /// Maximum count of warmup iterations that should be performed + /// The default value is 50 + /// + public int? MaxWarmupIterationCount { get; set; } + + /// + /// specifies whether Engine should allocate some random-sized memory between iterations + /// it makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration + /// + public bool? MemoryRandomization { get; set; } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Models/BdnHostInfo.cs b/src/BenchmarkDotNet/Models/BdnHostInfo.cs new file mode 100644 index 0000000000..ca33e4d8ef --- /dev/null +++ b/src/BenchmarkDotNet/Models/BdnHostInfo.cs @@ -0,0 +1,14 @@ +using Perfolizer.Models; + +namespace BenchmarkDotNet.Models; + +internal class BdnHostInfo : HostInfo +{ + public string RuntimeVersion { get; set; } = ""; + public bool HasAttachedDebugger { get; set; } + public bool HasRyuJit { get; set; } + public string Configuration { get; set; } = ""; + public string DotNetSdkVersion { get; set; } = ""; + public double ChronometerFrequency { get; set; } + public string HardwareTimerKind { get; set; } = ""; +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Models/BdnLifecycle.cs b/src/BenchmarkDotNet/Models/BdnLifecycle.cs new file mode 100644 index 0000000000..a93a325977 --- /dev/null +++ b/src/BenchmarkDotNet/Models/BdnLifecycle.cs @@ -0,0 +1,55 @@ +using System; +using BenchmarkDotNet.Engines; +using JetBrains.Annotations; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Models; + +internal class BdnLifecycle : LifecycleInfo, IEquatable, IComparable +{ + public int LaunchIndex { get; set; } + public IterationStage IterationStage { get; set; } = IterationStage.Unknown; + public IterationMode IterationMode { get; set; } = IterationMode.Unknown; + + public bool Equals(BdnLifecycle? other) + { + if (ReferenceEquals(null, other)) + return false; + if (ReferenceEquals(this, other)) + return true; + return LaunchIndex == other.LaunchIndex && + IterationMode == other.IterationMode && + IterationStage == other.IterationStage; + } + + public override bool Equals(object? obj) + { + if (ReferenceEquals(null, obj)) + return false; + if (ReferenceEquals(this, obj)) + return true; + if (obj.GetType() != GetType()) + return false; + return Equals((BdnLifecycle)obj); + } + + public override int GetHashCode() => HashCode.Combine(LaunchIndex, IterationMode, IterationStage); + + public static bool operator ==(BdnLifecycle? left, BdnLifecycle? right) => Equals(left, right); + public static bool operator !=(BdnLifecycle? left, BdnLifecycle? right) => !Equals(left, right); + + public int CompareTo(BdnLifecycle? other) + { + if (ReferenceEquals(this, other)) + return 0; + if (ReferenceEquals(null, other)) + return 1; + int launchIndexComparison = LaunchIndex.CompareTo(other.LaunchIndex); + if (launchIndexComparison != 0) + return launchIndexComparison; + int iterationStageComparison = IterationStage.CompareTo(other.IterationStage); + if (iterationStageComparison != 0) + return iterationStageComparison; + return IterationMode.CompareTo(other.IterationMode); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Models/BdnSchema.cs b/src/BenchmarkDotNet/Models/BdnSchema.cs new file mode 100644 index 0000000000..3006b702de --- /dev/null +++ b/src/BenchmarkDotNet/Models/BdnSchema.cs @@ -0,0 +1,17 @@ +using Perfolizer.Perfonar.Base; + +namespace BenchmarkDotNet.Models; + +internal class BdnSchema : PerfonarSchema +{ + public static readonly BdnSchema Instance = new(); + + private BdnSchema() : base("bdn") + { + Add(); + Add(); + Add(); + Add(); + Add(); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Order/CategoryComparer.cs b/src/BenchmarkDotNet/Order/CategoryComparer.cs index a64fb7c588..ac11695a59 100644 --- a/src/BenchmarkDotNet/Order/CategoryComparer.cs +++ b/src/BenchmarkDotNet/Order/CategoryComparer.cs @@ -7,7 +7,7 @@ namespace BenchmarkDotNet.Order internal class CategoryComparer : IComparer { private const string Separator = "§"; - public static readonly CategoryComparer Instance = new (); + public static readonly CategoryComparer Instance = new(); public int Compare(string[] x, string[] y) { diff --git a/src/BenchmarkDotNet/Order/DefaultOrderer.cs b/src/BenchmarkDotNet/Order/DefaultOrderer.cs index 656de7ce98..d0dc2d475b 100644 --- a/src/BenchmarkDotNet/Order/DefaultOrderer.cs +++ b/src/BenchmarkDotNet/Order/DefaultOrderer.cs @@ -19,7 +19,7 @@ public class DefaultOrderer : IOrderer private readonly IComparer categoryComparer = CategoryComparer.Instance; private readonly IComparer paramsComparer = ParameterComparer.Instance; - private readonly IComparer jobComparer = JobComparer.Instance; + private readonly IComparer jobComparer; private readonly IComparer targetComparer; public SummaryOrderPolicy SummaryOrderPolicy { get; } @@ -27,10 +27,14 @@ public class DefaultOrderer : IOrderer public DefaultOrderer( SummaryOrderPolicy summaryOrderPolicy = SummaryOrderPolicy.Default, - MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared) + MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared, + JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default) { SummaryOrderPolicy = summaryOrderPolicy; MethodOrderPolicy = methodOrderPolicy; + jobComparer = jobOrderPolicy == JobOrderPolicy.Default + ? JobComparer.Instance + : JobComparer.Numeric; targetComparer = new DescriptorComparer(methodOrderPolicy); } diff --git a/src/BenchmarkDotNet/Order/JobOrderPolicy.cs b/src/BenchmarkDotNet/Order/JobOrderPolicy.cs new file mode 100644 index 0000000000..be1d66186d --- /dev/null +++ b/src/BenchmarkDotNet/Order/JobOrderPolicy.cs @@ -0,0 +1,14 @@ +namespace BenchmarkDotNet.Order; + +public enum JobOrderPolicy +{ + /// + /// Compare job characteristics in ordinal order. + /// + Default, + + /// + /// Compare job characteristics in numeric order. + /// + Numeric, +} diff --git a/src/BenchmarkDotNet/Parameters/SmartParamBuilder.cs b/src/BenchmarkDotNet/Parameters/SmartParamBuilder.cs index 29814d8179..49ea1b162f 100644 --- a/src/BenchmarkDotNet/Parameters/SmartParamBuilder.cs +++ b/src/BenchmarkDotNet/Parameters/SmartParamBuilder.cs @@ -99,8 +99,20 @@ public string ToSourceCode() ? $"[{argumentIndex}]" // IEnumerable : string.Empty; // IEnumerable + string methodCall; + if ((source as MethodInfo)?.IsStatic ?? (source as PropertyInfo)?.GetMethod.IsStatic ?? throw new Exception($"{nameof(source)} was not {nameof(MethodInfo)} nor {nameof(PropertyInfo)}")) + { + // If the source member is static, we need to place the fully qualified type name before it, in case the source member is from another type that this generated type does not inherit from. + methodCall = $"{source.DeclaringType.GetCorrectCSharpTypeName()}.{source.Name}"; + } + else + { + // If the source member is non-static, we mustn't include the type name, as this would be a compiler error when accessing a non-static source member in the base class of this generated type. + methodCall = source.Name; + } + // we do something like enumerable.ElementAt(sourceIndex)[argumentIndex]; - return $"{cast}BenchmarkDotNet.Parameters.ParameterExtractor.GetParameter({source.Name}{callPostfix}, {sourceIndex}){indexPostfix};"; + return $"{cast}BenchmarkDotNet.Parameters.ParameterExtractor.GetParameter({methodCall}{callPostfix}, {sourceIndex}){indexPostfix};"; } } diff --git a/src/BenchmarkDotNet/Portability/Cpu/CpuInfo.cs b/src/BenchmarkDotNet/Portability/Cpu/CpuInfo.cs deleted file mode 100644 index 8c3227a25d..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/CpuInfo.cs +++ /dev/null @@ -1,37 +0,0 @@ -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Portability.Cpu -{ - public class CpuInfo - { - public string ProcessorName { get; } - public int? PhysicalProcessorCount { get; } - public int? PhysicalCoreCount { get; } - public int? LogicalCoreCount { get; } - public Frequency? NominalFrequency { get; } - public Frequency? MinFrequency { get; } - public Frequency? MaxFrequency { get; } - - internal CpuInfo(string processorName, Frequency? nominalFrequency) - : this(processorName, null, null, null, nominalFrequency, null, null) - { - } - - public CpuInfo(string processorName, - int? physicalProcessorCount, - int? physicalCoreCount, - int? logicalCoreCount, - Frequency? nominalFrequency, - Frequency? minFrequency, - Frequency? maxFrequency) - { - ProcessorName = processorName; - PhysicalProcessorCount = physicalProcessorCount; - PhysicalCoreCount = physicalCoreCount; - LogicalCoreCount = logicalCoreCount; - NominalFrequency = nominalFrequency; - MinFrequency = minFrequency; - MaxFrequency = maxFrequency; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/CpuInfoFormatter.cs b/src/BenchmarkDotNet/Portability/Cpu/CpuInfoFormatter.cs deleted file mode 100644 index e016ad1a4b..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/CpuInfoFormatter.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System.Collections.Generic; -using BenchmarkDotNet.Environments; - -namespace BenchmarkDotNet.Portability.Cpu -{ - public static class CpuInfoFormatter - { - public static string Format(CpuInfo cpuInfo) - { - if (cpuInfo == null) - { - return "Unknown processor"; - } - - var parts = new List - { - ProcessorBrandStringHelper.Prettify(cpuInfo, includeMaxFrequency: true) - }; - - if (cpuInfo.PhysicalProcessorCount > 0) - parts.Add($", {cpuInfo.PhysicalProcessorCount} CPU"); - - if (cpuInfo.LogicalCoreCount == 1) - parts.Add(", 1 logical core"); - - if (cpuInfo.LogicalCoreCount > 1) - parts.Add($", {cpuInfo.LogicalCoreCount} logical cores"); - - if (cpuInfo.LogicalCoreCount > 0 && cpuInfo.PhysicalCoreCount > 0) - parts.Add(" and "); - else if (cpuInfo.PhysicalCoreCount > 0) - parts.Add(", "); - - if (cpuInfo.PhysicalCoreCount == 1) - parts.Add("1 physical core"); - if (cpuInfo.PhysicalCoreCount > 1) - parts.Add($"{cpuInfo.PhysicalCoreCount} physical cores"); - - string result = string.Join("", parts); - // The line with ProcessorBrandString is one of the longest lines in the summary. - // When people past in on GitHub, it can be a reason of an ugly horizontal scrollbar. - // To avoid this, we are trying to minimize this line and use the minimum possible number of characters. - // Here we are removing the repetitive "cores" word. - if (result.Contains("logical cores") && result.Contains("physical cores")) - result = result.Replace("logical cores", "logical"); - - return result; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/MosCpuInfoProvider.cs b/src/BenchmarkDotNet/Portability/Cpu/MosCpuInfoProvider.cs deleted file mode 100644 index 2edc7cabcb..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/MosCpuInfoProvider.cs +++ /dev/null @@ -1,56 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Management; -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class MosCpuInfoProvider - { -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatform("windows")] -#endif - internal static readonly Lazy MosCpuInfo = new Lazy(Load); - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatform("windows")] -#endif - private static CpuInfo Load() - { - var processorModelNames = new HashSet(); - uint physicalCoreCount = 0; - uint logicalCoreCount = 0; - int processorsCount = 0; - uint nominalClockSpeed = 0; - uint maxClockSpeed = 0; - uint minClockSpeed = 0; - - - using (var mosProcessor = new ManagementObjectSearcher("SELECT * FROM Win32_Processor")) - { - foreach (var moProcessor in mosProcessor.Get().Cast()) - { - string name = moProcessor[WmicCpuInfoKeyNames.Name]?.ToString(); - if (!string.IsNullOrEmpty(name)) - { - processorModelNames.Add(name); - processorsCount++; - physicalCoreCount += (uint) moProcessor[WmicCpuInfoKeyNames.NumberOfCores]; - logicalCoreCount += (uint) moProcessor[WmicCpuInfoKeyNames.NumberOfLogicalProcessors]; - maxClockSpeed = (uint) moProcessor[WmicCpuInfoKeyNames.MaxClockSpeed]; - } - } - } - - return new CpuInfo( - processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null, - processorsCount > 0 ? processorsCount : (int?) null, - physicalCoreCount > 0 ? (int?) physicalCoreCount : null, - logicalCoreCount > 0 ? (int?) logicalCoreCount : null, - nominalClockSpeed > 0 && logicalCoreCount > 0 ? Frequency.FromMHz(nominalClockSpeed) : (Frequency?) null, - minClockSpeed > 0 && logicalCoreCount > 0 ? Frequency.FromMHz(minClockSpeed) : (Frequency?) null, - maxClockSpeed > 0 && logicalCoreCount > 0 ? Frequency.FromMHz(maxClockSpeed) : (Frequency?) null); - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoKeyNames.cs b/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoKeyNames.cs deleted file mode 100644 index 60284c5363..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoKeyNames.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class ProcCpuInfoKeyNames - { - internal const string PhysicalId = "physical id"; - internal const string CpuCores = "cpu cores"; - internal const string ModelName = "model name"; - internal const string MaxFrequency = "max freq"; - internal const string MinFrequency = "min freq"; - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoParser.cs b/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoParser.cs deleted file mode 100644 index e3d0a14c46..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoParser.cs +++ /dev/null @@ -1,73 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; -using BenchmarkDotNet.Helpers; -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class ProcCpuInfoParser - { - internal static CpuInfo ParseOutput(string? content) - { - var logicalCores = SectionsHelper.ParseSections(content, ':'); - var processorModelNames = new HashSet(); - var processorsToPhysicalCoreCount = new Dictionary(); - - int logicalCoreCount = 0; - var nominalFrequency = Frequency.Zero; - var minFrequency = Frequency.Zero; - var maxFrequency = Frequency.Zero; - - foreach (var logicalCore in logicalCores) - { - if (logicalCore.TryGetValue(ProcCpuInfoKeyNames.PhysicalId, out string physicalId) && - logicalCore.TryGetValue(ProcCpuInfoKeyNames.CpuCores, out string cpuCoresValue) && - int.TryParse(cpuCoresValue, out int cpuCoreCount) && - cpuCoreCount > 0) - processorsToPhysicalCoreCount[physicalId] = cpuCoreCount; - - if (logicalCore.TryGetValue(ProcCpuInfoKeyNames.ModelName, out string modelName)) - { - processorModelNames.Add(modelName); - nominalFrequency = ParseFrequencyFromBrandString(modelName); - logicalCoreCount++; - } - - if (logicalCore.TryGetValue(ProcCpuInfoKeyNames.MinFrequency, out string minCpuFreqValue) - && Frequency.TryParseMHz(minCpuFreqValue, out var minCpuFreq)) - { - minFrequency = minCpuFreq; - } - - if (logicalCore.TryGetValue(ProcCpuInfoKeyNames.MaxFrequency, out string maxCpuFreqValue) - && Frequency.TryParseMHz(maxCpuFreqValue, out var maxCpuFreq)) - { - maxFrequency = maxCpuFreq; - } - } - - return new CpuInfo( - processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null, - processorsToPhysicalCoreCount.Count > 0 ? processorsToPhysicalCoreCount.Count : (int?) null, - processorsToPhysicalCoreCount.Count > 0 ? processorsToPhysicalCoreCount.Values.Sum() : (int?) null, - logicalCoreCount > 0 ? logicalCoreCount : (int?) null, - nominalFrequency > 0 ? nominalFrequency : (Frequency?) null, - minFrequency > 0 ? minFrequency : (Frequency?) null, - maxFrequency > 0 ? maxFrequency : (Frequency?) null); - } - - internal static Frequency ParseFrequencyFromBrandString(string brandString) - { - const string pattern = "(\\d.\\d+)GHz"; - var matches = Regex.Matches(brandString, pattern, RegexOptions.IgnoreCase); - if (matches.Count > 0 && matches[0].Groups.Count > 1) - { - string match = Regex.Matches(brandString, pattern, RegexOptions.IgnoreCase)[0].Groups[1].ToString(); - return Frequency.TryParseGHz(match, out var result) ? result : Frequency.Zero; - } - - return 0d; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoProvider.cs b/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoProvider.cs deleted file mode 100644 index 7cadad1ff2..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoProvider.cs +++ /dev/null @@ -1,75 +0,0 @@ -using System; -using System.Linq; -using System.Text; -using BenchmarkDotNet.Helpers; -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Portability.Cpu -{ - /// - /// CPU information from output of the `cat /proc/info` command. - /// Linux only. - /// - internal static class ProcCpuInfoProvider - { - internal static readonly Lazy ProcCpuInfo = new (Load); - - private static CpuInfo? Load() - { - if (RuntimeInformation.IsLinux()) - { - string content = ProcessHelper.RunAndReadOutput("cat", "/proc/cpuinfo") ?? ""; - string output = GetCpuSpeed() ?? ""; - content += output; - return ProcCpuInfoParser.ParseOutput(content); - } - return null; - } - - private static string? GetCpuSpeed() - { - try - { - string[]? output = ProcessHelper.RunAndReadOutput("/bin/bash", "-c \"lscpu | grep MHz\"")? - .Split('\n') - .SelectMany(x => x.Split(':')) - .ToArray(); - - return ParseCpuFrequencies(output); - } - catch (Exception) - { - return null; - } - } - - private static string? ParseCpuFrequencies(string[]? input) - { - // Example of output we trying to parse: - // - // CPU MHz: 949.154 - // CPU max MHz: 3200,0000 - // CPU min MHz: 800,0000 - - if (input == null) - return null; - - var output = new StringBuilder(); - for (int i = 0; i + 1 < input.Length; i += 2) - { - string name = input[i].Trim(); - string value = input[i + 1].Trim(); - - if (name.EqualsWithIgnoreCase("CPU min MHz")) - if (Frequency.TryParseMHz(value.Replace(',', '.'), out var minFrequency)) - output.Append($"\n{ProcCpuInfoKeyNames.MinFrequency}\t:{minFrequency.ToMHz()}"); - - if (name.EqualsWithIgnoreCase("CPU max MHz")) - if (Frequency.TryParseMHz(value.Replace(',', '.'), out var maxFrequency)) - output.Append($"\n{ProcCpuInfoKeyNames.MaxFrequency}\t:{maxFrequency.ToMHz()}"); - } - - return output.ToString(); - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoParser.cs b/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoParser.cs deleted file mode 100644 index 7e61bbd3ec..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoParser.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Extensions; - -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class SysctlCpuInfoParser - { - [SuppressMessage("ReSharper", "StringLiteralTypo")] - internal static CpuInfo ParseOutput(string? content) - { - var sysctl = SectionsHelper.ParseSection(content, ':'); - string processorName = sysctl.GetValueOrDefault("machdep.cpu.brand_string"); - var physicalProcessorCount = GetPositiveIntValue(sysctl, "hw.packages"); - var physicalCoreCount = GetPositiveIntValue(sysctl, "hw.physicalcpu"); - var logicalCoreCount = GetPositiveIntValue(sysctl, "hw.logicalcpu"); - var nominalFrequency = GetPositiveLongValue(sysctl, "hw.cpufrequency"); - var minFrequency = GetPositiveLongValue(sysctl, "hw.cpufrequency_min"); - var maxFrequency = GetPositiveLongValue(sysctl, "hw.cpufrequency_max"); - return new CpuInfo(processorName, physicalProcessorCount, physicalCoreCount, logicalCoreCount, nominalFrequency, minFrequency, maxFrequency); - } - - private static int? GetPositiveIntValue(Dictionary sysctl, string keyName) - { - if (sysctl.TryGetValue(keyName, out string value) && - int.TryParse(value, out int result) && - result > 0) - return result; - return null; - } - - private static long? GetPositiveLongValue(Dictionary sysctl, string keyName) - { - if (sysctl.TryGetValue(keyName, out string value) && - long.TryParse(value, out long result) && - result > 0) - return result; - return null; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoProvider.cs b/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoProvider.cs deleted file mode 100644 index 32a207c1c8..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoProvider.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using BenchmarkDotNet.Helpers; - -namespace BenchmarkDotNet.Portability.Cpu -{ - /// - /// CPU information from output of the `sysctl -a` command. - /// MacOSX only. - /// - internal static class SysctlCpuInfoProvider - { - internal static readonly Lazy SysctlCpuInfo = new Lazy(Load); - - private static CpuInfo? Load() - { - if (RuntimeInformation.IsMacOS()) - { - string content = ProcessHelper.RunAndReadOutput("sysctl", "-a"); - return SysctlCpuInfoParser.ParseOutput(content); - } - return null; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoKeyNames.cs b/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoKeyNames.cs deleted file mode 100644 index 9dd2209304..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoKeyNames.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class WmicCpuInfoKeyNames - { - internal const string NumberOfLogicalProcessors = "NumberOfLogicalProcessors"; - internal const string NumberOfCores = "NumberOfCores"; - internal const string Name = "Name"; - internal const string MaxClockSpeed = "MaxClockSpeed"; - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoParser.cs b/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoParser.cs deleted file mode 100644 index 441b1a8b10..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoParser.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System.Collections.Generic; -using BenchmarkDotNet.Helpers; -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class WmicCpuInfoParser - { - internal static CpuInfo ParseOutput(string? content) - { - var processors = SectionsHelper.ParseSections(content, '='); - - var processorModelNames = new HashSet(); - int physicalCoreCount = 0; - int logicalCoreCount = 0; - int processorsCount = 0; - - var currentClockSpeed = Frequency.Zero; - var maxClockSpeed = Frequency.Zero; - var minClockSpeed = Frequency.Zero; - - foreach (var processor in processors) - { - if (processor.TryGetValue(WmicCpuInfoKeyNames.NumberOfCores, out string numberOfCoresValue) && - int.TryParse(numberOfCoresValue, out int numberOfCores) && - numberOfCores > 0) - physicalCoreCount += numberOfCores; - - if (processor.TryGetValue(WmicCpuInfoKeyNames.NumberOfLogicalProcessors, out string numberOfLogicalValue) && - int.TryParse(numberOfLogicalValue, out int numberOfLogical) && - numberOfLogical > 0) - logicalCoreCount += numberOfLogical; - - if (processor.TryGetValue(WmicCpuInfoKeyNames.Name, out string name)) - { - processorModelNames.Add(name); - processorsCount++; - } - - if (processor.TryGetValue(WmicCpuInfoKeyNames.MaxClockSpeed, out string frequencyValue) - && int.TryParse(frequencyValue, out int frequency) - && frequency > 0) - { - maxClockSpeed += frequency; - } - } - - return new CpuInfo( - processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null, - processorsCount > 0 ? processorsCount : (int?) null, - physicalCoreCount > 0 ? physicalCoreCount : (int?) null, - logicalCoreCount > 0 ? logicalCoreCount : (int?) null, - currentClockSpeed > 0 && processorsCount > 0 ? Frequency.FromMHz(currentClockSpeed / processorsCount) : (Frequency?) null, - minClockSpeed > 0 && processorsCount > 0 ? Frequency.FromMHz(minClockSpeed / processorsCount) : (Frequency?) null, - maxClockSpeed > 0 && processorsCount > 0 ? Frequency.FromMHz(maxClockSpeed / processorsCount) : (Frequency?) null); - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoProvider.cs b/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoProvider.cs deleted file mode 100644 index d259200701..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoProvider.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System; -using System.IO; -using BenchmarkDotNet.Helpers; - -namespace BenchmarkDotNet.Portability.Cpu -{ - /// - /// CPU information from output of the `wmic cpu get Name, NumberOfCores, NumberOfLogicalProcessors /Format:List` command. - /// Windows only. - /// - internal static class WmicCpuInfoProvider - { - internal static readonly Lazy WmicCpuInfo = new (Load); - - private const string DefaultWmicPath = @"C:\Windows\System32\wbem\WMIC.exe"; - - private static CpuInfo? Load() - { - if (RuntimeInformation.IsWindows()) - { - const string argList = $"{WmicCpuInfoKeyNames.Name}, {WmicCpuInfoKeyNames.NumberOfCores}, " + - $"{WmicCpuInfoKeyNames.NumberOfLogicalProcessors}, {WmicCpuInfoKeyNames.MaxClockSpeed}"; - string wmicPath = File.Exists(DefaultWmicPath) ? DefaultWmicPath : "wmic"; - string content = ProcessHelper.RunAndReadOutput(wmicPath, $"cpu get {argList} /Format:List"); - return WmicCpuInfoParser.ParseOutput(content); - } - return null; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Libc.cs b/src/BenchmarkDotNet/Portability/Libc.cs index c9a3cf1322..036d7b584d 100644 --- a/src/BenchmarkDotNet/Portability/Libc.cs +++ b/src/BenchmarkDotNet/Portability/Libc.cs @@ -2,7 +2,9 @@ namespace BenchmarkDotNet.Portability { +#pragma warning disable CS8981 // The type name 'libc' only contains lower-cased ascii characters. Such names may become reserved for the language. internal static class libc +#pragma warning restore CS8981 { [DllImport(nameof(libc))] internal static extern int getppid(); diff --git a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs index 6064b24dfa..9b8348a125 100644 --- a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs +++ b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs @@ -2,20 +2,16 @@ using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; -using System.IO; using System.Linq; using System.Management; using System.Reflection; using System.Runtime.InteropServices; using System.Text.RegularExpressions; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Portability.Cpu; -using JetBrains.Annotations; -using Microsoft.Win32; using static System.Runtime.InteropServices.RuntimeInformation; -using RuntimeEnvironment = Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment; namespace BenchmarkDotNet.Portability { @@ -25,46 +21,36 @@ internal static class RuntimeInformation internal const string ReleaseConfigurationName = "RELEASE"; internal const string Unknown = "?"; + // Many of these checks allocate and/or are expensive to compute. We store the results in static readonly fields to keep Engine non-allocating. + // Static readonly fields are used instead of properties to avoid an extra getter method call that might not be tier1 jitted. + // This class is internal, so we don't need to expose these as properties. + /// /// returns true for both the old (implementation of .NET Framework) and new Mono (.NET 6+ flavour) /// - public static bool IsMono { get; } = Type.GetType("Mono.RuntimeStructs") != null; // it allocates a lot of memory, we need to check it once in order to keep Engine non-allocating! + public static readonly bool IsMono = Type.GetType("Mono.RuntimeStructs") != null; - public static bool IsOldMono { get; } = Type.GetType("Mono.Runtime") != null; + public static readonly bool IsOldMono = Type.GetType("Mono.Runtime") != null; - public static bool IsNewMono { get; } = IsMono && !IsOldMono; + public static readonly bool IsNewMono = IsMono && !IsOldMono; - public static bool IsFullFramework => + public static readonly bool IsFullFramework = #if NET6_0_OR_GREATER + // This could be const, but we want to avoid unreachable code warnings. false; #else FrameworkDescription.StartsWith(".NET Framework", StringComparison.OrdinalIgnoreCase); #endif - [PublicAPI] - public static bool IsNetNative => FrameworkDescription.StartsWith(".NET Native", StringComparison.OrdinalIgnoreCase); - - public static bool IsNetCore - => ((Environment.Version.Major >= 5) || FrameworkDescription.StartsWith(".NET Core", StringComparison.OrdinalIgnoreCase)) - && !string.IsNullOrEmpty(typeof(object).Assembly.Location); - - public static bool IsNativeAOT - => Environment.Version.Major >= 5 - && string.IsNullOrEmpty(typeof(object).Assembly.Location) // it's merged to a single .exe and .Location returns null - && !IsWasm; // Wasm also returns "" for assembly locations - #if NET6_0_OR_GREATER [System.Runtime.Versioning.SupportedOSPlatformGuard("browser")] -#endif - public static bool IsWasm => -#if NET6_0_OR_GREATER - OperatingSystem.IsBrowser(); + public static readonly bool IsWasm = OperatingSystem.IsBrowser(); #else - IsOSPlatform(OSPlatform.Create("BROWSER")); + public static readonly bool IsWasm = IsOSPlatform(OSPlatform.Create("BROWSER")); #endif #if NETSTANDARD2_0 - public static bool IsAot { get; } = IsAotMethod(); // This allocates, so we only want to call it once statically. + public static readonly bool IsAot = IsAotMethod() || FrameworkDescription.StartsWith(".NET Native", StringComparison.OrdinalIgnoreCase); private static bool IsAotMethod() { @@ -82,175 +68,34 @@ private static bool IsAotMethod() return false; } #else - public static bool IsAot => !System.Runtime.CompilerServices.RuntimeFeature.IsDynamicCodeCompiled; -#endif - - public static bool IsRunningInContainer => string.Equals(Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER"), "true"); - - internal static string ExecutableExtension => IsWindows() ? ".exe" : string.Empty; - - internal static string ScriptFileExtension => IsWindows() ? ".bat" : ".sh"; - - internal static string GetArchitecture() => GetCurrentPlatform().ToString(); - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("windows")] -#endif - internal static bool IsWindows() => -#if NET6_0_OR_GREATER - OperatingSystem.IsWindows(); // prefer linker-friendly OperatingSystem APIs -#else - IsOSPlatform(OSPlatform.Windows); -#endif - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("linux")] -#endif - internal static bool IsLinux() => -#if NET6_0_OR_GREATER - OperatingSystem.IsLinux(); -#else - IsOSPlatform(OSPlatform.Linux); -#endif - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("macos")] -#endif - internal static bool IsMacOS() => -#if NET6_0_OR_GREATER - OperatingSystem.IsMacOS(); -#else - IsOSPlatform(OSPlatform.OSX); -#endif - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("android")] -#endif - internal static bool IsAndroid() => -#if NET6_0_OR_GREATER - OperatingSystem.IsAndroid(); -#else - Type.GetType("Java.Lang.Object, Mono.Android") != null; + public static readonly bool IsAot = !System.Runtime.CompilerServices.RuntimeFeature.IsDynamicCodeCompiled; #endif -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("ios")] -#endif - internal static bool IsIOS() => -#if NET6_0_OR_GREATER - OperatingSystem.IsIOS(); -#else - Type.GetType("Foundation.NSObject, Xamarin.iOS") != null; -#endif - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("tvos")] -#endif - internal static bool IsTvOS() => -#if NET6_0_OR_GREATER - OperatingSystem.IsTvOS(); -#else - IsOSPlatform(OSPlatform.Create("TVOS")); -#endif - - public static string GetOsVersion() - { - if (IsMacOS()) - { - string systemVersion = ExternalToolsHelper.MacSystemProfilerData.Value.GetValueOrDefault("System Version") ?? ""; - string kernelVersion = ExternalToolsHelper.MacSystemProfilerData.Value.GetValueOrDefault("Kernel Version") ?? ""; - if (!string.IsNullOrEmpty(systemVersion) && !string.IsNullOrEmpty(kernelVersion)) - return OsBrandStringHelper.PrettifyMacOSX(systemVersion, kernelVersion); - } - - if (IsLinux()) - { - string? version = GetLinuxOsVersion(); - if (version != null && version.Trim() != "") - return version; - } - - string operatingSystem = RuntimeEnvironment.OperatingSystem; - string operatingSystemVersion = RuntimeEnvironment.OperatingSystemVersion; - - return OsBrandStringHelper.Prettify( - operatingSystem, - operatingSystemVersion, - GetWindowsUbr()); - } - - private static string? GetLinuxOsVersion() - { - if (!IsLinux()) - return null; - try - { - string version = LinuxOsReleaseHelper.GetNameByOsRelease(File.ReadAllLines("/etc/os-release")); - bool wsl = IsUnderWsl(); - return wsl ? version + " WSL" : version; - } - catch (Exception) - { - return null; - } - } + public static bool IsNetCore + => ((Environment.Version.Major >= 5) || FrameworkDescription.StartsWith(".NET Core", StringComparison.OrdinalIgnoreCase)) + && !IsAot; - private static bool IsUnderWsl() - { - if (!IsLinux()) - return false; - try - { - return File.Exists("/proc/sys/fs/binfmt_misc/WSLInterop"); // https://superuser.com/a/1749811 - } - catch (Exception) - { - return false; - } - } + public static bool IsNativeAOT + => Environment.Version.Major >= 5 + && IsAot + && !IsWasm && !IsMono; // Wasm and MonoAOTLLVM are also AOT - // TODO: Introduce a common util API for registry calls, use it also in BenchmarkDotNet.Toolchains.CsProj.GetCurrentVersionBasedOnWindowsRegistry - /// - /// On Windows, this method returns UBR (Update Build Revision) based on Registry. - /// Returns null if the value is not available - /// - /// - private static int? GetWindowsUbr() - { - if (IsWindows()) - { - try - { - using (var baseKey = RegistryKey.OpenBaseKey(RegistryHive.LocalMachine, RegistryView.Registry32)) - using (var ndpKey = baseKey.OpenSubKey(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion")) - { - if (ndpKey == null) - return null; - return Convert.ToInt32(ndpKey.GetValue("UBR")); - } - } - catch (Exception) - { - return null; - } - } - return null; - } + public static readonly bool IsTieredJitEnabled = + IsNetCore + && (Environment.Version.Major < 3 + // Disabled by default in netcoreapp2.X, check if it's enabled. + ? Environment.GetEnvironmentVariable("COMPlus_TieredCompilation") == "1" + || Environment.GetEnvironmentVariable("DOTNET_TieredCompilation") == "1" + || (AppContext.TryGetSwitch("System.Runtime.TieredCompilation", out bool isEnabled) && isEnabled) + // Enabled by default in netcoreapp3.0+, check if it's disabled. + : Environment.GetEnvironmentVariable("COMPlus_TieredCompilation") != "0" + && Environment.GetEnvironmentVariable("DOTNET_TieredCompilation") != "0" + && (!AppContext.TryGetSwitch("System.Runtime.TieredCompilation", out isEnabled) || isEnabled)); - internal static CpuInfo GetCpuInfo() - { - if (IsWindows() && IsFullFramework && !IsMono) - return MosCpuInfoProvider.MosCpuInfo.Value; - if (IsWindows()) - return WmicCpuInfoProvider.WmicCpuInfo.Value; - if (IsLinux()) - return ProcCpuInfoProvider.ProcCpuInfo.Value; - if (IsMacOS()) - return SysctlCpuInfoProvider.SysctlCpuInfo.Value; + public static readonly bool IsRunningInContainer = string.Equals(Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER"), "true"); - return null; - } + internal static string GetArchitecture() => GetCurrentPlatform().ToString(); internal static string GetRuntimeVersion() { @@ -315,28 +160,25 @@ internal static string GetRuntimeVersion() private static string GetNetCoreVersion() { - if (IsAndroid()) + if (OsDetector.IsAndroid()) { return $".NET {Environment.Version}"; } - else - { - var coreclrAssemblyInfo = FileVersionInfo.GetVersionInfo(typeof(object).GetTypeInfo().Assembly.Location); - var corefxAssemblyInfo = FileVersionInfo.GetVersionInfo(typeof(Regex).GetTypeInfo().Assembly.Location); - if (CoreRuntime.TryGetVersion(out var version) && version >= new Version(5, 0)) - { - // after the merge of dotnet/corefx and dotnet/coreclr into dotnet/runtime the version should always be the same - Debug.Assert(coreclrAssemblyInfo.FileVersion == corefxAssemblyInfo.FileVersion); - - return $".NET {version} ({coreclrAssemblyInfo.FileVersion})"; - } - else - { - string runtimeVersion = version != default ? version.ToString() : Unknown; + return CoreRuntime.TryGetVersion(out var version) && version.Major >= 5 + ? $".NET {version} ({GetDetailedVersion()})" + : $".NET Core {version?.ToString() ?? Unknown} ({GetDetailedVersion()})"; - return $".NET Core {runtimeVersion} (CoreCLR {coreclrAssemblyInfo.FileVersion}, CoreFX {corefxAssemblyInfo.FileVersion})"; - } + string GetDetailedVersion() + { + string coreclrLocation = typeof(object).GetTypeInfo().Assembly.Location; + // Single-file publish has empty assembly location. + if (string.IsNullOrEmpty(coreclrLocation)) + return CoreRuntime.GetVersionFromFrameworkDescription(); + // .Net Core 2.X has confusing FrameworkDescription like 4.6.X. + if (version?.Major >= 3) + return $"{CoreRuntime.GetVersionFromFrameworkDescription()}, {FileVersionInfo.GetVersionInfo(coreclrLocation).FileVersion}"; + return FileVersionInfo.GetVersionInfo(coreclrLocation).FileVersion; } } @@ -364,12 +206,13 @@ internal static Runtime GetCurrentRuntime() public static Platform GetCurrentPlatform() { // these are not part of .NET Standard 2.0, so we use hardcoded values taken from - // https://github.com/dotnet/runtime/blob/1a37caf773a3b857ccff49a31be3333d4fdc491f/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Architecture.cs#L9 + // https://github.com/dotnet/runtime/blob/080fcae7eaa8367abf7900e08ff2e52e3efea5bf/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Architecture.cs#L9 const Architecture Wasm = (Architecture)4; const Architecture S390x = (Architecture)5; const Architecture LoongArch64 = (Architecture)6; const Architecture Armv6 = (Architecture)7; const Architecture Ppc64le = (Architecture)8; + const Architecture RiscV64 = (Architecture)9; switch (ProcessArchitecture) { @@ -391,6 +234,8 @@ public static Platform GetCurrentPlatform() return Platform.Armv6; case Ppc64le: return Platform.Ppc64le; + case RiscV64: + return Platform.RiscV64; default: throw new ArgumentOutOfRangeException(); } @@ -416,14 +261,14 @@ internal static string GetJitInfo() { if (IsNativeAOT) return "NativeAOT"; - if (IsNetNative || IsAot) + if (IsAot) return "AOT"; if (IsMono || IsWasm) return ""; // There is no helpful information about JIT on Mono if (IsNetCore || HasRyuJit()) // CoreCLR supports only RyuJIT return "RyuJIT"; if (IsFullFramework) - return "LegacyJIT"; + return "LegacyJIT"; return Unknown; } @@ -476,7 +321,7 @@ public JitModule(string name, string version) internal static ICollection GetAntivirusProducts() { var products = new List(); - if (IsWindows()) + if (OsDetector.IsWindows()) { try { @@ -484,7 +329,7 @@ internal static ICollection GetAntivirusProducts() using (var data = wmi.Get()) foreach (var o in data) { - var av = (ManagementObject) o; + var av = (ManagementObject)o; if (av != null) { string name = av["displayName"].ToString(); @@ -502,11 +347,11 @@ internal static ICollection GetAntivirusProducts() return products; } - internal static VirtualMachineHypervisor GetVirtualMachineHypervisor() + internal static VirtualMachineHypervisor? GetVirtualMachineHypervisor() { - VirtualMachineHypervisor[] hypervisors = { HyperV.Default, VirtualBox.Default, VMware.Default }; + VirtualMachineHypervisor[] hypervisors = [HyperV.Default, VirtualBox.Default, VMware.Default]; - if (IsWindows()) + if (OsDetector.IsWindows()) { try { @@ -532,4 +377,4 @@ internal static VirtualMachineHypervisor GetVirtualMachineHypervisor() return null; } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs index 5327197aa5..bec0dbbc48 100644 --- a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs +++ b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs @@ -1,26 +1,17 @@ using System; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -#if RELEASE using BenchmarkDotNet.Properties; -#endif [assembly: Guid("cbba82d3-e650-407f-a0f0-767891d4f04c")] [assembly: CLSCompliant(true)] -#if RELEASE [assembly: InternalsVisibleTo("BenchmarkDotNet.Tests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] +[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotMemory,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] -#else -[assembly: InternalsVisibleTo("BenchmarkDotNet.Tests")] -[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests")] -[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows")] -[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace")] -[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning")] -[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks")] -#endif \ No newline at end of file +[assembly: InternalsVisibleTo("BenchmarkDotNet.TestAdapter,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] diff --git a/src/BenchmarkDotNet/Properties/BenchmarkDotNetInfo.cs b/src/BenchmarkDotNet/Properties/BenchmarkDotNetInfo.cs index fcb01a0055..0941fa1cc3 100644 --- a/src/BenchmarkDotNet/Properties/BenchmarkDotNetInfo.cs +++ b/src/BenchmarkDotNet/Properties/BenchmarkDotNetInfo.cs @@ -1,12 +1,15 @@ using System; using System.Reflection; using BenchmarkDotNet.Extensions; +using Perfolizer.Models; namespace BenchmarkDotNet.Properties { public class BenchmarkDotNetInfo { - private static readonly Lazy LazyInstance = new (() => + public const string BenchmarkDotNetCaption = "BenchmarkDotNet"; + + private static readonly Lazy LazyInstance = new(() => { var assembly = typeof(BenchmarkDotNetInfo).GetTypeInfo().Assembly; var assemblyVersion = assembly.GetName().Version; @@ -16,6 +19,12 @@ public class BenchmarkDotNetInfo public static BenchmarkDotNetInfo Instance { get; } = LazyInstance.Value; + public EngineInfo GetBdnEngineInfo() => new() + { + Name = BenchmarkDotNetCaption, + Version = BrandVersion + }; + public Version AssemblyVersion { get; } public string FullVersion { get; } diff --git a/src/BenchmarkDotNet/Reports/BenchmarkReport.cs b/src/BenchmarkDotNet/Reports/BenchmarkReport.cs index b63c5221be..05f2a3b13e 100644 --- a/src/BenchmarkDotNet/Reports/BenchmarkReport.cs +++ b/src/BenchmarkDotNet/Reports/BenchmarkReport.cs @@ -3,10 +3,14 @@ using System.Collections.Immutable; using System.Linq; using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Models; using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.Results; using JetBrains.Annotations; +using Perfolizer.Horology; +using Perfolizer.Models; namespace BenchmarkDotNet.Reports { @@ -22,11 +26,12 @@ public sealed class BenchmarkReport public IReadOnlyList ExecuteResults { get; } - public Statistics? ResultStatistics => resultStatistics ?? (resultStatistics = GetResultRuns().Any() - ? new Statistics(GetResultRuns().Select(r => r.GetAverageTime().Nanoseconds)) - : null); + public Statistics? ResultStatistics => resultStatistics ??= + GetResultRuns().Any() + ? new Statistics(GetResultRuns().Select(r => r.GetAverageTime().Nanoseconds)) + : null; - private Statistics resultStatistics; + private Statistics? resultStatistics = null; public BenchmarkReport( bool success, @@ -44,11 +49,60 @@ public BenchmarkReport( AllMeasurements = ExecuteResults.SelectMany((results, index) => results.Measurements).ToArray(); GcStats = ExecuteResults.Count > 0 ? ExecuteResults[ExecuteResults.Count - 1].GcStats : default; Metrics = metrics?.ToDictionary(metric => metric.Descriptor.Id) - ?? (IReadOnlyDictionary)ImmutableDictionary.Empty; + ?? (IReadOnlyDictionary)ImmutableDictionary.Empty; } public override string ToString() => $"{BenchmarkCase.DisplayInfo}, {AllMeasurements.Count} runs"; public IReadOnlyList GetResultRuns() => AllMeasurements.Where(r => r.Is(IterationMode.Workload, IterationStage.Result)).ToList(); + + internal EntryInfo ToPerfonar() + { + var entry = new EntryInfo + { + Benchmark = new BdnBenchmark + { + Display = BenchmarkCase.DisplayInfo, + Namespace = BenchmarkCase.Descriptor.Type.Namespace ?? "", + Type = FullNameProvider.GetTypeName(BenchmarkCase.Descriptor.Type), + Method = BenchmarkCase.Descriptor.WorkloadMethod.Name, + Parameters = BenchmarkCase.Parameters.PrintInfo, + HardwareIntrinsics = this.GetHardwareIntrinsicsInfo() + }, + Job = new JobInfo + { + Environment = BenchmarkCase.Job.Environment.ToPerfonar(), + Execution = BenchmarkCase.Job.Run.ToPerfonar() + } + }; + + var lifecycles = AllMeasurements.GroupBy(m => new BdnLifecycle + { + LaunchIndex = m.LaunchIndex, + IterationMode = m.IterationMode, + IterationStage = m.IterationStage + }).OrderBy(x => x.Key).ToList(); + foreach (var lifecycleGroup in lifecycles) + { + var measurementsEntry = new EntryInfo + { + Lifecycle = lifecycleGroup.Key + }; + + foreach (var measurement in lifecycleGroup.ToList()) + { + measurementsEntry.Add(new EntryInfo + { + IterationIndex = measurement.IterationIndex, + InvocationCount = measurement.Operations, + Value = measurement.Nanoseconds / measurement.Operations, + Unit = TimeUnit.Nanosecond + }); + } + entry.Add(measurementsEntry); + } + + return entry; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Reports/Measurement.cs b/src/BenchmarkDotNet/Reports/Measurement.cs index ddcac16a30..9a1245ddca 100644 --- a/src/BenchmarkDotNet/Reports/Measurement.cs +++ b/src/BenchmarkDotNet/Reports/Measurement.cs @@ -1,5 +1,7 @@ using System; +#if DEBUG using System.Diagnostics; +#endif using System.Globalization; using System.Linq; using System.Text; @@ -95,7 +97,7 @@ public override string ToString() builder.Append(NsSymbol); builder.Append(", "); - builder.Append(GetAverageTime().ToString(MainCultureInfo).ToAscii()); + builder.Append(GetAverageTime().ToDefaultString("0.0000").ToAscii()); builder.Append("/op"); return builder.ToString(); diff --git a/src/BenchmarkDotNet/Reports/Summary.cs b/src/BenchmarkDotNet/Reports/Summary.cs index 680ded2d5b..343a0301e9 100644 --- a/src/BenchmarkDotNet/Reports/Summary.cs +++ b/src/BenchmarkDotNet/Reports/Summary.cs @@ -12,6 +12,10 @@ using BenchmarkDotNet.Validators; using JetBrains.Annotations; using Perfolizer.Horology; +using Perfolizer.Models; +using Perfolizer.Perfonar; +using Perfolizer.Perfonar.Base; +using Perfolizer.Perfonar.Tables; namespace BenchmarkDotNet.Reports { @@ -62,7 +66,8 @@ public Summary( DisplayPrecisionManager = new DisplayPrecisionManager(this); Orderer = GetConfiguredOrdererOrDefaultOne(reports.Select(report => report.BenchmarkCase.Config)); - BenchmarksCases = Orderer.GetSummaryOrder(reports.Select(report => report.BenchmarkCase).ToImmutableArray(), this).ToImmutableArray(); // we sort it first + BenchmarksCases = + Orderer.GetSummaryOrder(reports.Select(report => report.BenchmarkCase).ToImmutableArray(), this).ToImmutableArray(); // we sort it first Reports = BenchmarksCases.Select(b => ReportMap[b]).ToImmutableArray(); // we use sorted collection to re-create reports list BaseliningStrategy = BaseliningStrategy.Create(BenchmarksCases); Style = (summaryStyle ?? GetConfiguredSummaryStyleOrDefaultOne(BenchmarksCases)).WithCultureInfo(cultureInfo); @@ -75,7 +80,7 @@ public Summary( /// /// Returns a report for the given benchmark or null if there is no a corresponded report. /// - public BenchmarkReport this[BenchmarkCase benchmarkCase] => ReportMap.GetValueOrDefault(benchmarkCase); + public BenchmarkReport? this[BenchmarkCase benchmarkCase] => ReportMap.GetValueOrDefault(benchmarkCase); public bool HasCriticalValidationErrors => ValidationErrors.Any(validationError => validationError.IsCritical); @@ -84,8 +89,10 @@ public Summary( public bool IsMultipleRuntimes => isMultipleRuntimes ??= BenchmarksCases.Length > 1 ? BenchmarksCases.Select(benchmark => benchmark.GetRuntime()).Distinct().Count() > 1 : false; - internal static Summary ValidationFailed(string title, string resultsDirectoryPath, string logFilePath, ImmutableArray? validationErrors = null) - => new Summary(title, ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, DefaultCultureInfo.Instance, validationErrors ?? ImmutableArray.Empty, ImmutableArray.Empty); + internal static Summary ValidationFailed(string title, string resultsDirectoryPath, string logFilePath, + ImmutableArray? validationErrors = null) + => new Summary(title, ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, + DefaultCultureInfo.Instance, validationErrors ?? ImmutableArray.Empty, ImmutableArray.Empty); internal static Summary Join(List summaries, ClockSpan clockSpan) => new Summary( @@ -108,7 +115,7 @@ internal static string BuildAllRuntimes(HostEnvironmentInfo hostEnvironmentInfo, foreach (var benchmarkReport in reports) { - string runtime = benchmarkReport.GetRuntimeInfo(); + string? runtime = benchmarkReport.GetRuntimeInfo(); if (runtime != null) { string jobId = benchmarkReport.BenchmarkCase.Job.ResolvedId; @@ -135,7 +142,7 @@ internal static string BuildAllRuntimes(HostEnvironmentInfo hostEnvironmentInfo, public bool IsBaseline(BenchmarkCase benchmarkCase) => BaseliningStrategy.IsBaseline(benchmarkCase); - public BenchmarkCase? GetBaseline(string logicalGroupKey) + public BenchmarkCase? GetBaseline(string? logicalGroupKey) => BenchmarksCases .Where(b => GetLogicalGroupKey(b) == logicalGroupKey) .FirstOrDefault(IsBaseline); @@ -152,16 +159,54 @@ private static IOrderer GetConfiguredOrdererOrDefaultOne(IEnumerable config.Orderer != DefaultOrderer.Instance) .Select(config => config.Orderer) .Distinct() - .SingleOrDefault() + .FirstOrDefault() ?? DefaultOrderer.Instance; private static SummaryStyle GetConfiguredSummaryStyleOrDefaultOne(ImmutableArray benchmarkCases) => benchmarkCases .Where(benchmark => benchmark.Config.SummaryStyle != SummaryStyle.Default - && benchmark.Config.SummaryStyle != null) // Paranoid +#nullable disable + // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract ConditionIsAlwaysTrueOrFalse + // TODO: remove this check once the nullability migration is finished + && benchmark.Config.SummaryStyle != null) // Paranoid +#nullable enable .Select(benchmark => benchmark.Config.SummaryStyle) .Distinct() - .SingleOrDefault() + .FirstOrDefault() ?? SummaryStyle.Default; + + internal PerfonarTableConfig GetDefaultTableConfig() + { + return new PerfonarTableConfig + { + ColumnDefinitions = + [ + new PerfonarColumnDefinition(".engine") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PerfonarColumnDefinition(".host.os") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PerfonarColumnDefinition(".host.cpu") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PerfonarColumnDefinition(".benchmark") { Cloud = "secondary" }, + new PerfonarColumnDefinition(".job") { Cloud = "secondary", Compressed = true }, + new PerfonarColumnDefinition("=center"), + new PerfonarColumnDefinition("=spread") + ] + }; + } + + // TODO: GcStats + internal EntryInfo ToPerfonar() + { + var root = new EntryInfo + { + Engine = new EngineInfo + { + Name = HostEnvironmentInfo.BenchmarkDotNetCaption, + Version = HostEnvironmentInfo.BenchmarkDotNetVersion, + }, + Host = HostEnvironmentInfo.ToPerfonar(), + }; + foreach (var benchmarkReport in Reports) + root.Add(benchmarkReport.ToPerfonar()); + return root; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Reports/SummaryStyle.cs b/src/BenchmarkDotNet/Reports/SummaryStyle.cs index 23451442df..56d030671b 100644 --- a/src/BenchmarkDotNet/Reports/SummaryStyle.cs +++ b/src/BenchmarkDotNet/Reports/SummaryStyle.cs @@ -3,6 +3,7 @@ using BenchmarkDotNet.Columns; using BenchmarkDotNet.Helpers; using Perfolizer.Horology; +using Perfolizer.Metrology; using static BenchmarkDotNet.Reports.SummaryTable.SummaryTableColumn; // ReSharper disable MemberCanBePrivate.Global @@ -20,9 +21,9 @@ public class SummaryStyle : IEquatable public bool PrintUnitsInContent { get; } public bool PrintZeroValuesInContent { get; } public int MaxParameterColumnWidth { get; } - public SizeUnit SizeUnit { get; } + public SizeUnit? SizeUnit { get; } internal SizeUnit CodeSizeUnit { get; } - public TimeUnit TimeUnit { get; } + public TimeUnit? TimeUnit { get; } public CultureInfo CultureInfo { get; } public RatioStyle RatioStyle { get; } @@ -30,7 +31,7 @@ public class SummaryStyle : IEquatable public TextJustification TextColumnJustification { get; } public TextJustification NumericColumnJustification { get; } - public SummaryStyle(CultureInfo? cultureInfo, bool printUnitsInHeader, SizeUnit sizeUnit, TimeUnit timeUnit, bool printUnitsInContent = true, + public SummaryStyle(CultureInfo? cultureInfo, bool printUnitsInHeader, SizeUnit? sizeUnit, TimeUnit? timeUnit, bool printUnitsInContent = true, bool printZeroValuesInContent = false, int maxParameterColumnWidth = DefaultMaxParameterColumnWidth, RatioStyle ratioStyle = RatioStyle.Value, TextJustification textColumnJustification = TextJustification.Left, TextJustification numericColumnJustification = TextJustification.Right) { diff --git a/src/BenchmarkDotNet/Reports/SummaryTable.cs b/src/BenchmarkDotNet/Reports/SummaryTable.cs index 64aae074ec..35e0e315fa 100644 --- a/src/BenchmarkDotNet/Reports/SummaryTable.cs +++ b/src/BenchmarkDotNet/Reports/SummaryTable.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Extensions; using JetBrains.Annotations; using Perfolizer.Horology; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Reports { diff --git a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs index dbe350b2c8..5c9250a579 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs @@ -198,7 +198,9 @@ IEnumerable GetDefinitions(Func((attribute, parameterType) => { - var paramsValues = GetValidValuesForParamsSource(type, attribute.Name); + var targetType = attribute.Type ?? type; + + var paramsValues = GetValidValuesForParamsSource(targetType, attribute.Name); return SmartParamBuilder.CreateForParams(parameterType, paramsValues.source, paramsValues.values); }); @@ -208,7 +210,7 @@ IEnumerable GetDefinitions(Func GetArgumentsDefinitions(MethodInfo benchmark, Type target, SummaryStyle summaryStyle) + private static IEnumerable GetArgumentsDefinitions(MethodInfo benchmark, Type benchmarkType, SummaryStyle summaryStyle) { var argumentsAttributes = benchmark.GetCustomAttributes(); int priority = argumentsAttributes.Select(attribute => attribute.Priority).Sum(); @@ -244,8 +246,9 @@ private static IEnumerable GetArgumentsDefinitions(MethodInf yield break; var argumentsSourceAttribute = benchmark.GetCustomAttribute(); + var targetType = argumentsSourceAttribute.Type ?? benchmarkType; - var valuesInfo = GetValidValuesForParamsSource(target, argumentsSourceAttribute.Name); + var valuesInfo = GetValidValuesForParamsSource(targetType, argumentsSourceAttribute.Name); for (int sourceIndex = 0; sourceIndex < valuesInfo.values.Length; sourceIndex++) yield return SmartParamBuilder.CreateForArguments(benchmark, parameterDefinitions, valuesInfo, sourceIndex, summaryStyle); } @@ -263,16 +266,7 @@ private static void AssertMethodIsAccessible(string methodType, MethodInfo metho { if (!methodInfo.IsPublic) throw new InvalidBenchmarkDeclarationException($"{methodType} method {methodInfo.Name} has incorrect access modifiers.\nMethod must be public."); - - var declaringType = methodInfo.DeclaringType; - - while (declaringType != null) - { - if (!declaringType.GetTypeInfo().IsPublic && !declaringType.GetTypeInfo().IsNestedPublic) - throw new InvalidBenchmarkDeclarationException($"{declaringType.FullName} containing {methodType} method {methodInfo.Name} has incorrect access modifiers.\nDeclaring type must be public."); - - declaringType = declaringType.DeclaringType; - } + /* Moved the code that verifies if DeclaringType of a given MethodInfo (a method) is publicly accessible to CompilationValidator */ } private static void AssertMethodIsNotGeneric(string methodType, MethodInfo methodInfo) @@ -312,25 +306,25 @@ private static object Map(object providedValue, Type type) return providedValue; } - private static (MemberInfo source, object[] values) GetValidValuesForParamsSource(Type parentType, string sourceName) + private static (MemberInfo source, object[] values) GetValidValuesForParamsSource(Type sourceType, string sourceName) { - var paramsSourceMethod = parentType.GetAllMethods().SingleOrDefault(method => method.Name == sourceName && method.IsPublic); + var paramsSourceMethod = sourceType.GetAllMethods().FirstOrDefault(method => method.Name == sourceName && method.IsPublic); if (paramsSourceMethod != default) return (paramsSourceMethod, ToArray( - paramsSourceMethod.Invoke(paramsSourceMethod.IsStatic ? null : Activator.CreateInstance(parentType), null), + paramsSourceMethod.Invoke(paramsSourceMethod.IsStatic ? null : Activator.CreateInstance(sourceType), null), paramsSourceMethod, - parentType)); + sourceType)); - var paramsSourceProperty = parentType.GetAllProperties().SingleOrDefault(property => property.Name == sourceName && property.GetMethod.IsPublic); + var paramsSourceProperty = sourceType.GetAllProperties().FirstOrDefault(property => property.Name == sourceName && property.GetMethod.IsPublic); if (paramsSourceProperty != default) return (paramsSourceProperty, ToArray( - paramsSourceProperty.GetValue(paramsSourceProperty.GetMethod.IsStatic ? null : Activator.CreateInstance(parentType)), + paramsSourceProperty.GetValue(paramsSourceProperty.GetMethod.IsStatic ? null : Activator.CreateInstance(sourceType)), paramsSourceProperty, - parentType)); + sourceType)); - throw new InvalidBenchmarkDeclarationException($"{parentType.Name} has no public, accessible method/property called {sourceName}, unable to read values for [ParamsSource]"); + throw new InvalidBenchmarkDeclarationException($"{sourceType.Name} has no public, accessible method/property called {sourceName}, unable to read values for [ParamsSource]"); } private static object[] ToArray(object sourceValue, MemberInfo memberInfo, Type type) diff --git a/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs b/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs index ada1b2b78e..4892de7f0c 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs @@ -47,8 +47,10 @@ public bool Equals(BenchmarkCase x, BenchmarkCase y) return false; if (AreDifferent(jobX.Infrastructure.Arguments, jobY.Infrastructure.Arguments)) // arguments can be anything (Mono runtime settings or MsBuild parameters) return false; +#pragma warning disable CS0618 // Type or member is obsolete if (AreDifferent(jobX.Infrastructure.NuGetReferences, jobY.Infrastructure.NuGetReferences)) return false; +#pragma warning restore CS0618 // Type or member is obsolete if (!jobX.Environment.Gc.Equals(jobY.Environment.Gc)) // GC settings are per .config/.csproj return false; @@ -81,8 +83,10 @@ public int GetHashCode(BenchmarkCase obj) hashCode.Add(job.Infrastructure.BuildConfiguration); foreach (var arg in job.Infrastructure.Arguments ?? Array.Empty()) hashCode.Add(arg); +#pragma warning disable CS0618 // Type or member is obsolete foreach (var reference in job.Infrastructure.NuGetReferences ?? Array.Empty()) hashCode.Add(reference); +#pragma warning restore CS0618 // Type or member is obsolete return hashCode.ToHashCode(); } diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 56849e7421..fefd88a906 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -8,6 +8,7 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; @@ -18,9 +19,9 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Mathematics; -using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.DotNetCli; using BenchmarkDotNet.Toolchains.Parameters; using BenchmarkDotNet.Toolchains.Results; using BenchmarkDotNet.Validators; @@ -37,14 +38,13 @@ internal static class BenchmarkRunnerClean internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) { - using var taskbarProgress = new TaskbarProgress(); - taskbarProgress.SetState(TaskbarProgressState.Indeterminate); + using var taskbarProgress = new TaskbarProgress(TaskbarProgressState.Indeterminate); var resolver = DefaultResolver; var artifactsToCleanup = new List(); var rootArtifactsFolderPath = GetRootArtifactsFolderPath(benchmarkRunInfos); - var maxTitleLength = RuntimeInformation.IsWindows() + var maxTitleLength = OsDetector.IsWindows() ? 254 - rootArtifactsFolderPath.Length : int.MaxValue; var title = GetTitle(benchmarkRunInfos, maxTitleLength); @@ -55,6 +55,7 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) using (var streamLogger = new StreamLogger(GetLogFileStreamWriter(benchmarkRunInfos, logFilePath))) { var compositeLogger = CreateCompositeLogger(benchmarkRunInfos, streamLogger); + using var wakeLock = WakeLock.Request(WakeLock.GetWakeLockType(benchmarkRunInfos), "BenchmarkDotNet Running Benchmarks", streamLogger); var eventProcessor = new CompositeEventProcessor(benchmarkRunInfos); eventProcessor.OnStartValidationStage(); @@ -70,24 +71,48 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) PrintValidationErrors(compositeLogger, validationErrors); + eventProcessor.OnEndValidationStage(); // Ensure that OnEndValidationStage() is called when a critical validation error exists. + if (validationErrors.Any(validationError => validationError.IsCritical)) return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath, validationErrors.ToImmutableArray()) }; - if (!supportedBenchmarks.Any(benchmarks => benchmarks.BenchmarksCases.Any())) - return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath) }; - - eventProcessor.OnEndValidationStage(); - int totalBenchmarkCount = supportedBenchmarks.Sum(benchmarkInfo => benchmarkInfo.BenchmarksCases.Length); int benchmarksToRunCount = totalBenchmarkCount - (idToResume + 1); // ids are indexed from 0 compositeLogger.WriteLineHeader("// ***** BenchmarkRunner: Start *****"); compositeLogger.WriteLineHeader($"// ***** Found {totalBenchmarkCount} benchmark(s) in total *****"); var globalChronometer = Chronometer.Start(); - var buildPartitions = BenchmarkPartitioner.CreateForBuild(supportedBenchmarks, resolver); eventProcessor.OnStartBuildStage(buildPartitions); - var buildResults = BuildInParallel(compositeLogger, rootArtifactsFolderPath, buildPartitions, in globalChronometer, eventProcessor); + + var sequentialBuildPartitions = buildPartitions.Where(partition => + partition.Benchmarks.Any(x => x.Config.Options.IsSet(ConfigOptions.DisableParallelBuild)) + // .Net SDK 8+ supports ArtifactsPath for proper parallel builds. + // Older SDKs may produce builds with incorrect bindings if more than 1 partition is built in parallel. + || (partition.RepresentativeBenchmarkCase.GetToolchain().Generator is DotNetCliGenerator + && partition.RepresentativeBenchmarkCase.GetRuntime().RuntimeMoniker.GetRuntimeVersion().Major < 8) + ) + .ToArray(); + var parallelBuildPartitions = buildPartitions.Except(sequentialBuildPartitions).ToArray(); + + var buildResults = new Dictionary(); + if (parallelBuildPartitions.Length > 0) + { + var results = BuildInParallel(compositeLogger, rootArtifactsFolderPath, parallelBuildPartitions, in globalChronometer, eventProcessor); + foreach (var kvp in results) + { + buildResults.Add(kvp.Key, kvp.Value); + } + } + + if (sequentialBuildPartitions.Length > 0) + { + var results = BuildSequential(compositeLogger, rootArtifactsFolderPath, sequentialBuildPartitions, in globalChronometer, eventProcessor); + foreach (var kvp in results) + { + buildResults.Add(kvp.Key, kvp.Value); + } + } var allBuildsHaveFailed = buildResults.Values.All(buildResult => !buildResult.IsBuildSuccess); @@ -329,14 +354,14 @@ private static void PrintSummary(ILogger logger, ImmutableConfig config, Summary if (columnWithLegends.Any()) maxNameWidth = Math.Max(maxNameWidth, columnWithLegends.Select(c => c.ColumnName.Length).Max()); if (effectiveTimeUnit != null) - maxNameWidth = Math.Max(maxNameWidth, effectiveTimeUnit.Name.ToString(cultureInfo).Length + 2); + maxNameWidth = Math.Max(maxNameWidth, effectiveTimeUnit.Abbreviation.ToString(cultureInfo).Length + 2); foreach (var column in columnWithLegends) logger.WriteLineHint($" {column.ColumnName.PadRight(maxNameWidth, ' ')} : {column.Legend}"); if (effectiveTimeUnit != null) - logger.WriteLineHint($" {("1 " + effectiveTimeUnit.Name).PadRight(maxNameWidth, ' ')} :" + - $" 1 {effectiveTimeUnit.Description} ({TimeUnit.Convert(1, effectiveTimeUnit, TimeUnit.Second).ToString("0.#########", summary.GetCultureInfo())} sec)"); + logger.WriteLineHint($" {("1 " + effectiveTimeUnit.Abbreviation).PadRight(maxNameWidth, ' ')} :" + + $" 1 {effectiveTimeUnit.FullName} ({TimeUnit.Convert(1, effectiveTimeUnit, TimeUnit.Second).ToString("0.#########", summary.GetCultureInfo())} sec)"); } if (config.GetDiagnosers().Any()) @@ -353,17 +378,6 @@ private static ImmutableArray Validate(params BenchmarkRunInfo[ { var validationErrors = new List(); - if (benchmarks.Any(b => b.Config.Options.IsSet(ConfigOptions.JoinSummary))) - { - var joinedCases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - validationErrors.AddRange( - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(joinedCases, null)) - ); - } - foreach (var benchmarkRunInfo in benchmarks) validationErrors.AddRange(benchmarkRunInfo.Config.GetCompositeValidator().Validate(new ValidationParameters(benchmarkRunInfo.BenchmarksCases, benchmarkRunInfo.Config))); @@ -418,11 +432,31 @@ private static Dictionary BuildInParallel(ILogger l logger.WriteLineHeader($"// ***** Done, took {GetFormattedDifference(afterParallelBuild, afterSequentialBuild)} *****"); return buildResults; + } - static string GetFormattedDifference(ClockSpan before, ClockSpan after) - => (after.GetTimeSpan() - before.GetTimeSpan()).ToFormattedTotalTime(DefaultCultureInfo.Instance); + private static Dictionary BuildSequential(ILogger logger, string rootArtifactsFolderPath, BuildPartition[] buildPartitions, in StartedClock globalChronometer, EventProcessor eventProcessor) + { + logger.WriteLineHeader($"// ***** Building {buildPartitions.Length} exe(s) in Sequential: Start *****"); + + var beforeBuild = globalChronometer.GetElapsed(); + + var buildResults = new Dictionary(); + foreach (var buildPartition in buildPartitions) + { + buildResults[buildPartition] = Build(buildPartition, rootArtifactsFolderPath, logger); + eventProcessor.OnBuildComplete(buildPartition, buildResults[buildPartition]); + } + + var afterBuild = globalChronometer.GetElapsed(); + + logger.WriteLineHeader($"// ***** Done, took {GetFormattedDifference(beforeBuild, afterBuild)} *****"); + + return buildResults; } + private static string GetFormattedDifference(ClockSpan before, ClockSpan after) + => (after.GetTimeSpan() - before.GetTimeSpan()).ToFormattedTotalTime(DefaultCultureInfo.Instance); + private static BuildResult Build(BuildPartition buildPartition, string rootArtifactsFolderPath, ILogger buildLogger) { var toolchain = buildPartition.RepresentativeBenchmarkCase.GetToolchain(); // it's guaranteed that all the benchmarks in single partition have same toolchain @@ -588,21 +622,48 @@ private static void LogTotalTime(ILogger logger, TimeSpan time, int executedBenc private static (BenchmarkRunInfo[], List) GetSupportedBenchmarks(BenchmarkRunInfo[] benchmarkRunInfos, IResolver resolver) { - List validationErrors = new (); + List validationErrors = new(); + List runInfos = new(benchmarkRunInfos.Length); + + if (benchmarkRunInfos.Length == 0) + { + validationErrors.Add(new ValidationError(true, $"No benchmarks were found.")); + return (Array.Empty(), validationErrors); + } + + foreach (var benchmarkRunInfo in benchmarkRunInfos) + { + if (benchmarkRunInfo.BenchmarksCases.Length == 0) + { + validationErrors.Add(new ValidationError(true, $"No [Benchmark] attribute found on '{benchmarkRunInfo.Type.Name}' benchmark case.")); + continue; + } - var runInfos = benchmarkRunInfos.Select(info => new BenchmarkRunInfo( - info.BenchmarksCases.Where(benchmark => + var validBenchmarks = benchmarkRunInfo.BenchmarksCases + .Where(benchmark => { - var errors = benchmark.GetToolchain().Validate(benchmark, resolver).ToArray(); + + var errors = benchmark.GetToolchain() + .Validate(benchmark, resolver) + .ToArray(); + validationErrors.AddRange(errors); - return !errors.Any(); - }).ToArray(), - info.Type, - info.Config)) - .Where(infos => infos.BenchmarksCases.Any()) - .ToArray(); - - return (runInfos, validationErrors); + + return !errors.Any(error => error.IsCritical); + }) + .ToArray(); + + runInfos.Add( + new BenchmarkRunInfo( + validBenchmarks, + benchmarkRunInfo.Type, + benchmarkRunInfo.Config + + )); + + + } + return (runInfos.ToArray(), validationErrors); } private static string GetRootArtifactsFolderPath(BenchmarkRunInfo[] benchmarkRunInfos) @@ -675,6 +736,9 @@ void AddLogger(ILogger logger) foreach (var logger in benchmarkRunInfo.Config.GetLoggers()) AddLogger(logger); + if (benchmarkRunInfos.Length == 0) + AddLogger(new ConsoleLogger()); + AddLogger(streamLogger); return new CompositeLogger(loggers.Values.ToImmutableHashSet()); diff --git a/src/BenchmarkDotNet/Running/BuildPartition.cs b/src/BenchmarkDotNet/Running/BuildPartition.cs index f89863215c..2ebe5ee44c 100644 --- a/src/BenchmarkDotNet/Running/BuildPartition.cs +++ b/src/BenchmarkDotNet/Running/BuildPartition.cs @@ -1,11 +1,17 @@ using System; using System.IO; using System.Reflection; +using System.Threading; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.DotNetCli; using BenchmarkDotNet.Toolchains.MonoWasm; using BenchmarkDotNet.Toolchains.Roslyn; using JetBrains.Annotations; @@ -14,12 +20,16 @@ namespace BenchmarkDotNet.Running { public class BuildPartition { + // We use an auto-increment global counter instead of Guid to guarantee uniqueness per benchmark run (Guid has a small chance to collide), + // assuming there are fewer than 4 billion build partitions (a safe assumption). + internal static int s_partitionCounter; + public BuildPartition(BenchmarkBuildInfo[] benchmarks, IResolver resolver) { Resolver = resolver; RepresentativeBenchmarkCase = benchmarks[0].BenchmarkCase; Benchmarks = benchmarks; - ProgramName = benchmarks[0].Config.Options.IsSet(ConfigOptions.KeepBenchmarkFiles) ? RepresentativeBenchmarkCase.Job.FolderInfo : Guid.NewGuid().ToString(); + ProgramName = GetProgramName(RepresentativeBenchmarkCase, Interlocked.Increment(ref s_partitionCounter)); LogBuildOutput = benchmarks[0].Config.Options.IsSet(ConfigOptions.LogBuildOutput); GenerateMSBuildBinLog = benchmarks[0].Config.Options.IsSet(ConfigOptions.GenerateMSBuildBinLog); } @@ -71,5 +81,47 @@ private static string GetResolvedAssemblyLocation(Assembly assembly) => // in case of SingleFile, location.Length returns 0, so we use GetName() and // manually construct the path. assembly.Location.Length == 0 ? Path.Combine(AppContext.BaseDirectory, assembly.GetName().Name) : assembly.Location; + + internal static string GetProgramName(BenchmarkCase representativeBenchmarkCase, int id) + { + // Combine the benchmark's assembly name, folder info, and build partition id. + string benchmarkAssemblyName = representativeBenchmarkCase.Descriptor.Type.Assembly.GetName().Name; + string folderInfo = representativeBenchmarkCase.Job.FolderInfo; + var programName = $"{benchmarkAssemblyName}-{folderInfo}-{id}"; + // Very long program name can cause the path to exceed Windows' 260 character limit, + // for example BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks. + // 36 is an arbitrary limit, but it's the length of Guid strings which is what was used previously. + const int MaxLength = 36; + if (!OsDetector.IsWindows() || programName.Length <= MaxLength) + { + return programName; + } + programName = $"{benchmarkAssemblyName}-{id}"; + if (programName.Length <= MaxLength) + { + return programName; + } + programName = $"{folderInfo}-{id}"; + if (programName.Length <= MaxLength) + { + return programName; + } + return id.ToString(); + } + + internal bool ForcedNoDependenciesForIntegrationTests + { + get + { + if (!XUnitHelper.IsIntegrationTest.Value || !RuntimeInformation.IsNetCore) + return false; + + var job = RepresentativeBenchmarkCase.Job; + if (job.GetToolchain().Builder is not DotNetCliBuilder) + return false; + + return !job.HasDynamicBuildCharacteristic(); + } + } } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Running/ConsoleTitler.cs b/src/BenchmarkDotNet/Running/ConsoleTitler.cs index 9f8e21a1fe..d045e498aa 100644 --- a/src/BenchmarkDotNet/Running/ConsoleTitler.cs +++ b/src/BenchmarkDotNet/Running/ConsoleTitler.cs @@ -1,6 +1,7 @@ using System; using System.IO; -using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Helpers; namespace BenchmarkDotNet.Running { @@ -8,7 +9,7 @@ namespace BenchmarkDotNet.Running /// Updates Console.Title, subject to platform capabilities and Console availability. /// Restores the original (or fallback) title upon disposal. /// - internal class ConsoleTitler : IDisposable + internal class ConsoleTitler : DisposeAtProcessTermination { /// /// Whether this instance has any effect. This will be false if the platform doesn't support Console retitling, @@ -62,7 +63,7 @@ public ConsoleTitler(string initialTitle) #if NET6_0_OR_GREATER [System.Runtime.Versioning.SupportedOSPlatformGuard("windows")] #endif - private static bool PlatformSupportsTitleRead() => RuntimeInformation.IsWindows(); + private static bool PlatformSupportsTitleRead() => OsDetector.IsWindows(); /// /// Updates Console.Title if enabled. @@ -75,13 +76,14 @@ public void UpdateTitle(string title) } } - public void Dispose() + public override void Dispose() { if (IsEnabled) { Console.Title = oldConsoleTitle; IsEnabled = false; } + base.Dispose(); } } } diff --git a/src/BenchmarkDotNet/Running/Descriptor.cs b/src/BenchmarkDotNet/Running/Descriptor.cs index a38b1a13d9..4ff74fced7 100644 --- a/src/BenchmarkDotNet/Running/Descriptor.cs +++ b/src/BenchmarkDotNet/Running/Descriptor.cs @@ -11,10 +11,10 @@ public class Descriptor : IEquatable { public Type Type { get; } public MethodInfo WorkloadMethod { get; } - public MethodInfo GlobalSetupMethod { get; } - public MethodInfo GlobalCleanupMethod { get; } - public MethodInfo IterationSetupMethod { get; } - public MethodInfo IterationCleanupMethod { get; } + public MethodInfo? GlobalSetupMethod { get; } + public MethodInfo? GlobalCleanupMethod { get; } + public MethodInfo? IterationSetupMethod { get; } + public MethodInfo? IterationCleanupMethod { get; } public string AdditionalLogic { get; } public int OperationsPerInvoke { get; } public string WorkloadMethodDisplayInfo { get; } @@ -22,10 +22,10 @@ public class Descriptor : IEquatable public bool Baseline { get; } public string[] Categories { get; } - internal string TypeInfo => Type?.GetDisplayName() ?? "Untitled"; - private string MethodFolderInfo => WorkloadMethod?.Name ?? "Untitled"; + internal string TypeInfo => Type.GetDisplayName(); + private string MethodFolderInfo => WorkloadMethod.Name; - public string FolderInfo => (Type != null ? FolderNameHelper.ToFolderName(Type) : "Untitled") + "_" + MethodFolderInfo; + public string FolderInfo => $"{FolderNameHelper.ToFolderName(Type)}_{MethodFolderInfo}"; public string DisplayInfo => TypeInfo + "." + WorkloadMethodDisplayInfo; public Descriptor( @@ -42,6 +42,9 @@ public Descriptor( int operationsPerInvoke = 1, int methodIndex = 0) { + Assertion.NotNull(nameof(type), type); + Assertion.NotNull(nameof(workloadMethod), workloadMethod); + Type = type; WorkloadMethod = workloadMethod; GlobalSetupMethod = globalSetupMethod; @@ -58,9 +61,9 @@ public Descriptor( public override string ToString() => DisplayInfo; - private static string FormatDescription(string? description) + private static string? FormatDescription(string? description) { - var specialSymbols = new[] { ' ', '\'', '[', ']' }; + char[] specialSymbols = { ' ', '\'', '[', ']' }; return description != null && specialSymbols.Any(description.Contains) ? "'" + description + "'" : description; @@ -70,9 +73,9 @@ private static string FormatDescription(string? description) public string GetFilterName() => $"{Type.GetCorrectCSharpTypeName(includeGenericArgumentsNamespace: false)}.{WorkloadMethod.Name}"; - public bool Equals(Descriptor other) => GetFilterName().Equals(other.GetFilterName()); + public bool Equals(Descriptor? other) => GetFilterName().Equals(other?.GetFilterName()); - public override bool Equals(object obj) => obj is Descriptor && Equals((Descriptor)obj); + public override bool Equals(object? obj) => obj is Descriptor descriptor && Equals(descriptor); public override int GetHashCode() => GetFilterName().GetHashCode(); } diff --git a/src/BenchmarkDotNet/Running/PowerManagementApplier.cs b/src/BenchmarkDotNet/Running/PowerManagementApplier.cs index fdfa0239f5..f48d95d61e 100644 --- a/src/BenchmarkDotNet/Running/PowerManagementApplier.cs +++ b/src/BenchmarkDotNet/Running/PowerManagementApplier.cs @@ -1,13 +1,13 @@ using System; using System.Collections.Generic; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Running { - internal class PowerManagementApplier : IDisposable + internal class PowerManagementApplier : DisposeAtProcessTermination { private static readonly Guid UserPowerPlan = new Guid("67b4a053-3646-4532-affd-0535c9ea82a7"); @@ -27,13 +27,17 @@ internal class PowerManagementApplier : IDisposable internal PowerManagementApplier(ILogger logger) => this.logger = logger; - public void Dispose() => ApplyUserPowerPlan(); + public override void Dispose() + { + ApplyUserPowerPlan(); + base.Dispose(); + } internal static Guid Map(PowerPlan value) => PowerPlansDict[value]; internal void ApplyPerformancePlan(Guid id) { - if (!RuntimeInformation.IsWindows() || id == Guid.Empty) + if (!OsDetector.IsWindows() || id == Guid.Empty) return; if (id != UserPowerPlan) @@ -44,7 +48,7 @@ internal void ApplyPerformancePlan(Guid id) private void ApplyUserPowerPlan() { - if (powerPlanChanged && RuntimeInformation.IsWindows()) + if (powerPlanChanged && OsDetector.IsWindows()) { try { diff --git a/src/BenchmarkDotNet/Running/TypeFilter.cs b/src/BenchmarkDotNet/Running/TypeFilter.cs index 4e58ebdeaf..4d2188844f 100644 --- a/src/BenchmarkDotNet/Running/TypeFilter.cs +++ b/src/BenchmarkDotNet/Running/TypeFilter.cs @@ -15,6 +15,32 @@ public static (bool allTypesValid, IReadOnlyList runnable) GetTypesWithRun { var validRunnableTypes = new List(); + bool hasRunnableTypeBenchmarks = types.Any(type => type.ContainsRunnableBenchmarks()); + bool hasRunnableAssemblyBenchmarks = assemblies.Any(assembly => GenericBenchmarksBuilder.GetRunnableBenchmarks(assembly.GetRunnableBenchmarks()).Length > 0); + + if (!hasRunnableTypeBenchmarks && !hasRunnableAssemblyBenchmarks) + { + if (types.Any()) + { + foreach (var type in types) + { + logger.WriteLineError($"No [Benchmark] attribute found on '{type.Name}' benchmark case."); + } + } + else if (assemblies.Any()) + { + foreach (var assembly in assemblies) + { + logger.WriteLineError($"No [Benchmark] attribute found on '{assembly.GetName().Name}' assembly."); + } + } + else + { + logger.WriteLineError("No benchmarks were found."); + } + return (false, Array.Empty()); + } + foreach (var type in types) { if (type.ContainsRunnableBenchmarks()) diff --git a/src/BenchmarkDotNet/Running/WakeLock.PInvoke.cs b/src/BenchmarkDotNet/Running/WakeLock.PInvoke.cs new file mode 100644 index 0000000000..be002d5d20 --- /dev/null +++ b/src/BenchmarkDotNet/Running/WakeLock.PInvoke.cs @@ -0,0 +1,110 @@ +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; + +namespace BenchmarkDotNet.Running; + +internal partial class WakeLock +{ + private static class PInvoke + { + public static SafePowerHandle PowerCreateRequest(string reason) + { + IntPtr reasonPtr = Marshal.StringToHGlobalAuto(reason); + try + { + REASON_CONTEXT context = new REASON_CONTEXT() + { + Version = POWER_REQUEST_CONTEXT_VERSION, + Flags = POWER_REQUEST_CONTEXT_FLAGS.POWER_REQUEST_CONTEXT_SIMPLE_STRING, + Reason = new REASON_CONTEXT.REASON_CONTEXT_UNION { SimpleReasonString = reasonPtr } + + }; + SafePowerHandle safePowerHandle = PowerCreateRequest(context); + if (safePowerHandle.IsInvalid) { throw new Win32Exception(); } + return safePowerHandle; + } + finally + { + if (reasonPtr != IntPtr.Zero) + { + Marshal.FreeHGlobal(reasonPtr); + } + } + } + + [DllImport("kernel32.dll", ExactSpelling = true, SetLastError = true)] + private static extern SafePowerHandle PowerCreateRequest(REASON_CONTEXT Context); + + public static void PowerSetRequest(SafePowerHandle safePowerHandle, POWER_REQUEST_TYPE requestType) + { + if (!InvokePowerSetRequest(safePowerHandle, requestType)) + { + throw new Win32Exception(); + } + } + + [DllImport("kernel32.dll", EntryPoint = "PowerSetRequest", ExactSpelling = true, SetLastError = true)] + private static extern bool InvokePowerSetRequest(SafePowerHandle PowerRequest, POWER_REQUEST_TYPE RequestType); + + public static void PowerClearRequest(SafePowerHandle safePowerHandle, POWER_REQUEST_TYPE requestType) + { + if (!InvokePowerClearRequest(safePowerHandle, requestType)) + { + throw new Win32Exception(); + } + } + + [DllImport("kernel32.dll", EntryPoint = "PowerClearRequest", ExactSpelling = true, SetLastError = true)] + private static extern bool InvokePowerClearRequest(SafePowerHandle PowerRequest, POWER_REQUEST_TYPE RequestType); + + [DllImport("kernel32.dll", ExactSpelling = true, SetLastError = true)] + public static extern bool CloseHandle(nint hObject); + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] + private struct REASON_CONTEXT + { + public uint Version; + + public POWER_REQUEST_CONTEXT_FLAGS Flags; + + public REASON_CONTEXT_UNION Reason; + + [StructLayout(LayoutKind.Explicit, CharSet = CharSet.Unicode)] + public struct REASON_CONTEXT_UNION + { + [FieldOffset(0)] + public nint SimpleReasonString; + + // The DETAILED structure is not (yet) used, but needed for ARM CPUs, otherwise PowerCreateRequest fails, see #2745 + [FieldOffset(0)] + public DETAILED Detailed; + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] + public struct DETAILED + { + public nint LocalizedReasonModule; + public uint LocalizedReasonId; + public uint ReasonStringCount; + public nint ReasonStrings; + } + } + } + + private const uint POWER_REQUEST_CONTEXT_VERSION = 0U; + + private enum POWER_REQUEST_CONTEXT_FLAGS : uint + { + POWER_REQUEST_CONTEXT_DETAILED_STRING = 2U, + POWER_REQUEST_CONTEXT_SIMPLE_STRING = 1U, + } + + public enum POWER_REQUEST_TYPE + { + PowerRequestDisplayRequired = 0, + PowerRequestSystemRequired = 1, + PowerRequestAwayModeRequired = 2, + PowerRequestExecutionRequired = 3, + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/WakeLock.SafePowerHandle.cs b/src/BenchmarkDotNet/Running/WakeLock.SafePowerHandle.cs new file mode 100644 index 0000000000..8e6203574d --- /dev/null +++ b/src/BenchmarkDotNet/Running/WakeLock.SafePowerHandle.cs @@ -0,0 +1,13 @@ +using Microsoft.Win32.SafeHandles; + +namespace BenchmarkDotNet.Running; + +internal partial class WakeLock +{ + private sealed class SafePowerHandle : SafeHandleZeroOrMinusOneIsInvalid + { + private SafePowerHandle() : base(true) { } + + protected override bool ReleaseHandle() => PInvoke.CloseHandle(handle); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/WakeLock.cs b/src/BenchmarkDotNet/Running/WakeLock.cs new file mode 100644 index 0000000000..124fbd5e22 --- /dev/null +++ b/src/BenchmarkDotNet/Running/WakeLock.cs @@ -0,0 +1,69 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Loggers; +using System; +using System.ComponentModel; +using System.Linq; + +namespace BenchmarkDotNet.Running; + +internal partial class WakeLock +{ + public static WakeLockType GetWakeLockType(BenchmarkRunInfo[] benchmarkRunInfos) => + benchmarkRunInfos.Length == 0 ? WakeLockType.None : benchmarkRunInfos.Select(static i => i.Config.WakeLock).Max(); + + private static readonly bool OsVersionIsSupported = + // Must be windows 7 or greater + OsDetector.IsWindows() && Environment.OSVersion.Version >= new Version(6, 1); + + public static IDisposable? Request(WakeLockType wakeLockType, string reason, ILogger logger) => + wakeLockType == WakeLockType.None || !OsVersionIsSupported ? null : new WakeLockSentinel(wakeLockType, reason, logger); + + private class WakeLockSentinel : DisposeAtProcessTermination + { + private readonly WakeLockType wakeLockType; + private readonly SafePowerHandle? safePowerHandle; + private readonly ILogger logger; + + public WakeLockSentinel(WakeLockType wakeLockType, string reason, ILogger logger) + { + this.wakeLockType = wakeLockType; + this.logger = logger; + try + { + safePowerHandle = PInvoke.PowerCreateRequest(reason); + PInvoke.PowerSetRequest(safePowerHandle, PInvoke.POWER_REQUEST_TYPE.PowerRequestSystemRequired); + if (wakeLockType == WakeLockType.Display) + { + PInvoke.PowerSetRequest(safePowerHandle, PInvoke.POWER_REQUEST_TYPE.PowerRequestDisplayRequired); + } + } + catch (Win32Exception ex) + { + logger.WriteLineError($"Unable to prevent the system from entering sleep or turning off the display (error message: {ex.Message})."); + } + } + + public override void Dispose() + { + if (safePowerHandle != null) + { + try + { + if (wakeLockType == WakeLockType.Display) + { + PInvoke.PowerClearRequest(safePowerHandle, PInvoke.POWER_REQUEST_TYPE.PowerRequestDisplayRequired); + } + PInvoke.PowerClearRequest(safePowerHandle, PInvoke.POWER_REQUEST_TYPE.PowerRequestSystemRequired); + } + catch (Win32Exception ex) + { + logger.WriteLineError($"Unable to allow the system from entering sleep or turning off the display (error message: {ex.Message})."); + } + safePowerHandle.Dispose(); + } + base.Dispose(); + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Templates/BenchmarkType.txt b/src/BenchmarkDotNet/Templates/BenchmarkType.txt index f17737d646..ea1bbaa858 100644 --- a/src/BenchmarkDotNet/Templates/BenchmarkType.txt +++ b/src/BenchmarkDotNet/Templates/BenchmarkType.txt @@ -3,7 +3,7 @@ { public static void Run(BenchmarkDotNet.Engines.IHost host, System.String benchmarkName) { - BenchmarkDotNet.Autogenerated.Runnable_$ID$ instance = new BenchmarkDotNet.Autogenerated.Runnable_$ID$(); // do NOT change name "instance" (used in SmartParamameter) + BenchmarkDotNet.Autogenerated.Runnable_$ID$ instance = new BenchmarkDotNet.Autogenerated.Runnable_$ID$ { $ParamsInitializer$ }; // do NOT change name "instance" (used in SmartParamameter) $ParamsContent$ host.WriteLine(); diff --git a/src/BenchmarkDotNet/Templates/BuildPlots.R b/src/BenchmarkDotNet/Templates/BuildPlots.R index da071d8d41..c5dd6118f6 100644 --- a/src/BenchmarkDotNet/Templates/BuildPlots.R +++ b/src/BenchmarkDotNet/Templates/BuildPlots.R @@ -105,7 +105,7 @@ for (file in files) { densityPlot <- ggplot(df, aes(x=Measurement_Value, fill=Job_Id)) + ggtitle(paste(title, "/", target)) + xlab(paste("Time,", timeUnit)) + - geom_density(alpha=.5, bw="SJ") + geom_density(alpha=.5, bw="sj") printNice(densityPlot) ggsaveNice(gsub("-measurements.csv", paste0("-", target, "-density.png"), file), densityPlot) @@ -118,7 +118,7 @@ for (file in files) { paramsDensityPlot <- ggplot(paramsDf, aes(x=Measurement_Value, fill=Job_Id)) + ggtitle(paste(title, "/", target, "/", params)) + xlab(paste("Time,", timeUnit)) + - geom_density(alpha=.5, bw="SJ") + geom_density(alpha=.5, bw="sj") printNice(paramsDensityPlot) prefix <- createPrefix(c(target,params)) ggsaveNice(gsub("-measurements.csv", paste0(prefix, "-density.png"), file), paramsDensityPlot) @@ -155,7 +155,7 @@ for (file in files) { densityPlotJob <- ggplot(jobDf, aes(x=Measurement_Value, fill="red")) + ggtitle(paste(title, "/", target, "/", job)) + xlab(paste("Time,", timeUnit)) + - geom_density(alpha=.5, bw="SJ") + geom_density(alpha=.5, bw="sj") printNice(densityPlotJob) ggsaveNice(gsub("-measurements.csv", paste0("-", target, "-", job, "-density.png"), file), densityPlotJob) } diff --git a/src/BenchmarkDotNet/Templates/CsProj.txt b/src/BenchmarkDotNet/Templates/CsProj.txt index 5cd08250dc..99e536b985 100644 --- a/src/BenchmarkDotNet/Templates/CsProj.txt +++ b/src/BenchmarkDotNet/Templates/CsProj.txt @@ -9,7 +9,6 @@ $PLATFORM$ $PROGRAMNAME$ Exe - bin\$CONFIGURATIONNAME$ False false @@ -23,9 +22,10 @@ true latest - true false + + true BenchmarkDotNet.Autogenerated.UniqueProgramName diff --git a/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt b/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt index 1db40072d1..adcb87389e 100644 --- a/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt +++ b/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt @@ -9,7 +9,6 @@ Exe - bin $TFM$ $RUNTIMEPACK$ $RUNTIMEIDENTIFIER$ @@ -20,6 +19,8 @@ true BenchmarkDotNet.Autogenerated.UniqueProgramName true + + true diff --git a/src/BenchmarkDotNet/Templates/WasmCsProj.txt b/src/BenchmarkDotNet/Templates/WasmCsProj.txt index 1910911fe0..3db39f5669 100644 --- a/src/BenchmarkDotNet/Templates/WasmCsProj.txt +++ b/src/BenchmarkDotNet/Templates/WasmCsProj.txt @@ -11,20 +11,22 @@ Exe - bin Release false $TFM$ true - $(MSBuildThisFileDirectory)\bin\$TFM$\browser-wasm\publish + $(PublishDir) $PROGRAMNAME$ browser-wasm true $RUN_AOT$ $(RunAOTCompilation) true + false false false + + true BenchmarkDotNet.Autogenerated.UniqueProgramName @@ -48,8 +50,8 @@ - - + + diff --git a/src/BenchmarkDotNet/Toolchains/AppConfigGenerator.cs b/src/BenchmarkDotNet/Toolchains/AppConfigGenerator.cs index c1db452d2b..fcaf3824b8 100644 --- a/src/BenchmarkDotNet/Toolchains/AppConfigGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/AppConfigGenerator.cs @@ -8,6 +8,8 @@ using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; +#nullable enable + namespace BenchmarkDotNet.Toolchains { internal static class AppConfigGenerator @@ -26,22 +28,32 @@ internal static class AppConfigGenerator internal static void Generate(Job job, TextReader source, TextWriter destination, IResolver resolver) { - using (var xmlReader = XmlReader.Create(source)) - { - var xmlDocument = new XmlDocument(); + var xmlDocument = new XmlDocument(); + + XmlNode configurationElement; - var configurationElement = GetOrCreateConfigurationElement(xmlDocument, xmlReader); + if (source == TextReader.Null) + { + // Create a new configuration node. + configurationElement = xmlDocument.CreateNode(XmlNodeType.Element, "configuration", string.Empty); + xmlDocument.AppendChild(configurationElement); + } + else + { + // Try to get configuration node from specified TextReader. + using var xmlReader = XmlReader.Create(source); + configurationElement = GetOrCreateConfigurationElement(xmlDocument, xmlReader); + } - var runtimeElement = GetOrCreateRuntimeElement(xmlDocument, configurationElement); + var runtimeElement = GetOrCreateRuntimeElement(xmlDocument, configurationElement); - ClearStartupSettingsForCustomClr(configurationElement, job.Environment.Runtime); - ClearAllRuntimeSettingsThatCanBeSetOnlyByJobConfiguration(runtimeElement); + ClearStartupSettingsForCustomClr(configurationElement, job.Environment.Runtime); + ClearAllRuntimeSettingsThatCanBeSetOnlyByJobConfiguration(runtimeElement); - GenerateJitSettings(xmlDocument, runtimeElement, job.Environment); - GenerateGCSettings(xmlDocument, runtimeElement, job.Environment.Gc, resolver); + GenerateJitSettings(xmlDocument, runtimeElement, job.Environment); + GenerateGCSettings(xmlDocument, runtimeElement, job.Environment.Gc, resolver); - xmlDocument.Save(destination); - } + xmlDocument.Save(destination); } private static XmlNode GetOrCreateConfigurationElement(XmlDocument xmlDocument, XmlReader xmlReader) @@ -49,19 +61,23 @@ private static XmlNode GetOrCreateConfigurationElement(XmlDocument xmlDocument, try { xmlDocument.Load(xmlReader); - - return xmlDocument.SelectSingleNode("/configuration"); + var configurationNode = xmlDocument.SelectSingleNode("/configuration"); + if (configurationNode != null) + return configurationNode; } - catch // empty document + catch (XmlException) { - return xmlDocument.AppendChild(xmlDocument.CreateNode(XmlNodeType.Element, "configuration", string.Empty)); + // Failed to load XML content. } + + // If the XML is invalid or configuration node is not exists. Create a new configuration element + return xmlDocument.AppendChild(xmlDocument.CreateNode(XmlNodeType.Element, "configuration", string.Empty))!; } private static XmlNode GetOrCreateRuntimeElement(XmlDocument xmlDocument, XmlNode configurationElement) { return configurationElement.SelectSingleNode("runtime") - ?? configurationElement.AppendChild(xmlDocument.CreateNode(XmlNodeType.Element, "runtime", string.Empty)); + ?? configurationElement.AppendChild(xmlDocument.CreateNode(XmlNodeType.Element, "runtime", string.Empty))!; } private static void ClearAllRuntimeSettingsThatCanBeSetOnlyByJobConfiguration(XmlNode runtimeElement) @@ -75,7 +91,7 @@ private static void ClearAllRuntimeSettingsThatCanBeSetOnlyByJobConfiguration(Xm } } - private static void ClearStartupSettingsForCustomClr(XmlNode configurationElement, Runtime runtime) + private static void ClearStartupSettingsForCustomClr(XmlNode configurationElement, Runtime? runtime) { if (!(runtime is ClrRuntime clrRuntime) || string.IsNullOrEmpty(clrRuntime.Version)) return; diff --git a/src/BenchmarkDotNet/Toolchains/ArtifactsPaths.cs b/src/BenchmarkDotNet/Toolchains/ArtifactsPaths.cs index 72faf5c956..a9467494fa 100644 --- a/src/BenchmarkDotNet/Toolchains/ArtifactsPaths.cs +++ b/src/BenchmarkDotNet/Toolchains/ArtifactsPaths.cs @@ -4,11 +4,12 @@ namespace BenchmarkDotNet.Toolchains { public class ArtifactsPaths { - public static readonly ArtifactsPaths Empty = new ArtifactsPaths("", "", "", "", "", "", "", "", "", "", ""); + public static readonly ArtifactsPaths Empty = new ArtifactsPaths("", "", "", "", "", "", "", "", "", "", "", ""); [PublicAPI] public string RootArtifactsFolderPath { get; } [PublicAPI] public string BuildArtifactsDirectoryPath { get; } [PublicAPI] public string BinariesDirectoryPath { get; } + [PublicAPI] public string PublishDirectoryPath { get; } [PublicAPI] public string ProgramCodePath { get; } [PublicAPI] public string AppConfigPath { get; } [PublicAPI] public string NuGetConfigPath { get; } @@ -22,6 +23,7 @@ public ArtifactsPaths( string rootArtifactsFolderPath, string buildArtifactsDirectoryPath, string binariesDirectoryPath, + string publishDirectoryPath, string programCodePath, string appConfigPath, string nuGetConfigPath, @@ -34,6 +36,7 @@ public ArtifactsPaths( RootArtifactsFolderPath = rootArtifactsFolderPath; BuildArtifactsDirectoryPath = buildArtifactsDirectoryPath; BinariesDirectoryPath = binariesDirectoryPath; + PublishDirectoryPath = publishDirectoryPath; ProgramCodePath = programCodePath; AppConfigPath = appConfigPath; NuGetConfigPath = nuGetConfigPath; diff --git a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs index 7eca3f97f0..3cbaa80f65 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs @@ -15,12 +15,12 @@ public class CoreRunToolchain : IToolchain /// /// the path to CoreRun /// /should a copy of CoreRun be performed? True by default. The toolchain replaces old dependencies in CoreRun folder with newer versions if used by the benchmarks. - /// TFM, netcoreapp2.1 is the default + /// TFM, net8.0 is the default /// path to dotnet cli, if not provided the one from PATH will be used /// display name, CoreRun is the default value /// the directory to restore packages to public CoreRunToolchain(FileInfo coreRun, bool createCopy = true, - string targetFrameworkMoniker = "netcoreapp2.1", + string targetFrameworkMoniker = "net8.0", FileInfo? customDotNetCliPath = null, DirectoryInfo? restorePath = null, string displayName = "CoreRun") { @@ -66,7 +66,7 @@ public IEnumerable Validate(BenchmarkCase benchmark, IResolver $"Provided CoreRun path does not exist, benchmark '{benchmark.DisplayInfo}' will not be executed. Please remember that BDN expects path to CoreRun.exe (corerun on Unix), not to Core_Root folder.", benchmark); } - else if (Toolchain.IsCliPathInvalid(CustomDotNetCliPath?.FullName, benchmark, out var invalidCliError)) + else if (DotNetSdkValidator.IsCliPathInvalid(CustomDotNetCliPath?.FullName, benchmark, out var invalidCliError)) { yield return invalidCliError; } diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs index e3c3f85125..2761bd7c2c 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.DotNetCli; @@ -44,16 +45,22 @@ public override IEnumerable Validate(BenchmarkCase benchmarkCas yield return validationError; } - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) { yield return new ValidationError(true, $"Classic .NET toolchain is supported only for Windows, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", benchmarkCase); + yield break; } - else if (IsCliPathInvalid(CustomDotNetCliPath, benchmarkCase, out var invalidCliError)) + else if (DotNetSdkValidator.IsCliPathInvalid(CustomDotNetCliPath, benchmarkCase, out var invalidCliError)) { yield return invalidCliError; } + + foreach (var validationError in DotNetSdkValidator.ValidateFrameworkSdks(benchmarkCase)) + { + yield return validationError; + } } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs index 98bcd2ae8b..4aea1327f8 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs @@ -25,6 +25,7 @@ public class CsProjCoreToolchain : Toolchain, IEquatable [PublicAPI] public static readonly IToolchain NetCoreApp70 = From(NetCoreAppSettings.NetCoreApp70); [PublicAPI] public static readonly IToolchain NetCoreApp80 = From(NetCoreAppSettings.NetCoreApp80); [PublicAPI] public static readonly IToolchain NetCoreApp90 = From(NetCoreAppSettings.NetCoreApp90); + [PublicAPI] public static readonly IToolchain NetCoreApp10_0 = From(NetCoreAppSettings.NetCoreApp10_0); internal CsProjCoreToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) : base(name, generator, builder, executor) @@ -49,11 +50,6 @@ public override IEnumerable Validate(BenchmarkCase benchmarkCas yield return validationError; } - if (IsCliPathInvalid(CustomDotNetCliPath, benchmarkCase, out var invalidCliError)) - { - yield return invalidCliError; - } - if (benchmarkCase.Job.HasValue(EnvironmentMode.JitCharacteristic) && benchmarkCase.Job.ResolveValue(EnvironmentMode.JitCharacteristic, resolver) == Jit.LegacyJit) { yield return new ValidationError(true, @@ -80,6 +76,11 @@ public override IEnumerable Validate(BenchmarkCase benchmarkCas $"Currently CsProjCoreToolchain does not support LINQPad 6+. Please use {nameof(InProcessEmitToolchain)} instead.", benchmarkCase); } + + foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(CustomDotNetCliPath, benchmarkCase)) + { + yield return validationError; + } } public override bool Equals(object obj) => obj is CsProjCoreToolchain typed && Equals(typed); diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs index eaf1da4683..7b5a642744 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs @@ -82,7 +82,6 @@ protected override void GenerateProject(BuildPartition buildPartition, Artifacts .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) .Replace("$RUNTIMESETTINGS$", GetRuntimeSettings(benchmark.Job.Environment.Gc, buildPartition.Resolver)) .Replace("$COPIEDSETTINGS$", customProperties) - .Replace("$CONFIGURATIONNAME$", buildPartition.BuildConfiguration) .Replace("$SDKNAME$", sdkName) .ToString(); diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/CustomDotNetCliToolchainBuilder.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/CustomDotNetCliToolchainBuilder.cs index dc48965f1e..6223f9ea3c 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/CustomDotNetCliToolchainBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/CustomDotNetCliToolchainBuilder.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Portability; using JetBrains.Annotations; @@ -49,7 +50,7 @@ public CustomDotNetCliToolchainBuilder UseNuGetClearTag(bool value) return this; } - /// TFM, example: netcoreapp2.1 + /// TFM, example: net8.0 [PublicAPI] [SuppressMessage("ReSharper", "ParameterHidesMember")] public CustomDotNetCliToolchainBuilder TargetFrameworkMoniker(string targetFrameworkMoniker) @@ -127,7 +128,7 @@ internal static string GetPortableRuntimeIdentifier() // Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment.GetRuntimeIdentifier() // returns win10-x64, we want the simpler form win-x64 // the values taken from https://docs.microsoft.com/en-us/dotnet/core/rid-catalog#macos-rids - string osPart = RuntimeInformation.IsWindows() ? "win" : (RuntimeInformation.IsMacOS() ? "osx" : "linux"); + string osPart = OsDetector.IsWindows() ? "win" : (OsDetector.IsMacOS() ? "osx" : "linux"); string architecture = #if NETSTANDARD diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs index e034279f89..7a7afc3184 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs @@ -14,15 +14,13 @@ public class DotNetCliBuilder : IBuilder private string CustomDotNetCliPath { get; } private bool LogOutput { get; } - private bool RetryFailedBuildWithNoDeps { get; } [PublicAPI] - public DotNetCliBuilder(string targetFrameworkMoniker, string? customDotNetCliPath = null, bool logOutput = false, bool retryFailedBuildWithNoDeps = true) + public DotNetCliBuilder(string targetFrameworkMoniker, string? customDotNetCliPath = null, bool logOutput = false) { TargetFrameworkMoniker = targetFrameworkMoniker; CustomDotNetCliPath = customDotNetCliPath; LogOutput = logOutput; - RetryFailedBuildWithNoDeps = retryFailedBuildWithNoDeps; } public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger) @@ -35,8 +33,7 @@ public BuildResult Build(GenerateResult generateResult, BuildPartition buildPart buildPartition, Array.Empty(), buildPartition.Timeout, - logOutput: LogOutput, - retryFailedBuildWithNoDeps: RetryFailedBuildWithNoDeps) + logOutput: LogOutput) .RestoreThenBuild(); if (buildResult.IsBuildSuccess && buildPartition.RepresentativeBenchmarkCase.Job.Environment.LargeAddressAware) diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs index 9967b4ef92..70f959121e 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs @@ -1,10 +1,12 @@ using System; using System.Collections.Generic; +using System.ComponentModel; +using System.IO; using System.Linq; using System.Text; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; @@ -32,11 +34,8 @@ public class DotNetCliCommand [PublicAPI] public bool LogOutput { get; } - [PublicAPI] public bool RetryFailedBuildWithNoDeps { get; } - public DotNetCliCommand(string cliPath, string arguments, GenerateResult generateResult, ILogger logger, - BuildPartition buildPartition, IReadOnlyList environmentVariables, TimeSpan timeout, bool logOutput = false, - bool retryFailedBuildWithNoDeps = true) + BuildPartition buildPartition, IReadOnlyList environmentVariables, TimeSpan timeout, bool logOutput = false) { CliPath = cliPath ?? DotNetCliCommandExecutor.DefaultDotNetCliPath.Value; Arguments = arguments; @@ -46,21 +45,22 @@ public DotNetCliCommand(string cliPath, string arguments, GenerateResult generat EnvironmentVariables = environmentVariables; Timeout = timeout; LogOutput = logOutput || (buildPartition is not null && buildPartition.LogBuildOutput); - RetryFailedBuildWithNoDeps = retryFailedBuildWithNoDeps; } public DotNetCliCommand WithArguments(string arguments) - => new (CliPath, arguments, GenerateResult, Logger, BuildPartition, EnvironmentVariables, Timeout, logOutput: LogOutput); + => new(CliPath, arguments, GenerateResult, Logger, BuildPartition, EnvironmentVariables, Timeout, logOutput: LogOutput); public DotNetCliCommand WithCliPath(string cliPath) - => new (cliPath, Arguments, GenerateResult, Logger, BuildPartition, EnvironmentVariables, Timeout, logOutput: LogOutput); + => new(cliPath, Arguments, GenerateResult, Logger, BuildPartition, EnvironmentVariables, Timeout, logOutput: LogOutput); [PublicAPI] public BuildResult RestoreThenBuild() { DotNetCliCommandExecutor.LogEnvVars(WithArguments(null)); +#pragma warning disable CS0618 // Type or member is obsolete var packagesResult = AddPackages(); +#pragma warning restore CS0618 // Type or member is obsolete if (!packagesResult.IsSuccess) return BuildResult.Failure(GenerateResult, packagesResult.AllInformation); @@ -70,30 +70,29 @@ public BuildResult RestoreThenBuild() if (BuildPartition.IsCustomBuildConfiguration) return Build().ToBuildResult(GenerateResult); - var restoreResult = Restore(); - if (!restoreResult.IsSuccess) - return BuildResult.Failure(GenerateResult, restoreResult.AllInformation); - - // On our CI (Windows+.NET 7), Integration tests take to much time because each benchmark run rebuilds the BenchmarkDotNet itself. + // On our CI, Integration tests take too much time, because each benchmark run rebuilds BenchmarkDotNet itself. // To reduce the total duration of the CI workflows, we build all the projects without dependencies - bool forceNoDependencies = XUnitHelper.IsIntegrationTest.Value && - RuntimeInformation.IsWindows() && - RuntimeInformation.IsNetCore; - - DotNetCliCommandResult buildResult; - if (forceNoDependencies) - buildResult = BuildNoRestoreNoDependencies(); - else + if (BuildPartition.ForcedNoDependenciesForIntegrationTests) { - buildResult = BuildNoRestore(); - if (!buildResult.IsSuccess && RetryFailedBuildWithNoDeps) // if we fail to do the full build, we try with --no-dependencies - buildResult = BuildNoRestoreNoDependencies(); + var restoreResult = DotNetCliCommandExecutor.Execute(WithArguments( + GetRestoreCommand(GenerateResult.ArtifactsPaths, BuildPartition, $"{Arguments} --no-dependencies", "restore-no-deps", excludeOutput: true))); + if (!restoreResult.IsSuccess) + return BuildResult.Failure(GenerateResult, restoreResult.AllInformation); + + return DotNetCliCommandExecutor.Execute(WithArguments( + GetBuildCommand(GenerateResult.ArtifactsPaths, BuildPartition, $"{Arguments} --no-restore --no-dependencies", "build-no-restore-no-deps", excludeOutput: true))) + .ToBuildResult(GenerateResult); } + else + { + var restoreResult = Restore(); + if (!restoreResult.IsSuccess) + return BuildResult.Failure(GenerateResult, restoreResult.AllInformation); - if (!buildResult.IsSuccess) - return BuildResult.Failure(GenerateResult, buildResult.AllInformation); - - return buildResult.ToBuildResult(GenerateResult); + // We no longer retry with --no-dependencies, because it fails with --output set at the same time, + // and the artifactsPaths.BinariesDirectoryPath is set before we try to build, so we cannot overwrite it. + return BuildNoRestore().ToBuildResult(GenerateResult); + } } [PublicAPI] @@ -101,7 +100,9 @@ public BuildResult RestoreThenBuildThenPublish() { DotNetCliCommandExecutor.LogEnvVars(WithArguments(null)); +#pragma warning disable CS0618 // Type or member is obsolete var packagesResult = AddPackages(); +#pragma warning restore CS0618 // Type or member is obsolete if (!packagesResult.IsSuccess) return BuildResult.Failure(GenerateResult, packagesResult.AllInformation); @@ -115,16 +116,12 @@ public BuildResult RestoreThenBuildThenPublish() if (!restoreResult.IsSuccess) return BuildResult.Failure(GenerateResult, restoreResult.AllInformation); - var buildResult = BuildNoRestore(); - if (!buildResult.IsSuccess && RetryFailedBuildWithNoDeps) // if we fail to do the full build, we try with --no-dependencies - buildResult = BuildNoRestoreNoDependencies(); - - if (!buildResult.IsSuccess) - return BuildResult.Failure(GenerateResult, buildResult.AllInformation); - - return PublishNoBuildAndNoRestore().ToBuildResult(GenerateResult); + // We use the implicit build in the publish command. We stopped doing a separate build step because we set the --output. + return PublishNoRestore().ToBuildResult(GenerateResult); } + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This method will soon be removed")] public DotNetCliCommandResult AddPackages() { var executionTime = new TimeSpan(0); @@ -151,22 +148,20 @@ public DotNetCliCommandResult BuildNoRestore() => DotNetCliCommandExecutor.Execute(WithArguments( GetBuildCommand(GenerateResult.ArtifactsPaths, BuildPartition, $"{Arguments} --no-restore", "build-no-restore"))); - public DotNetCliCommandResult BuildNoRestoreNoDependencies() - => DotNetCliCommandExecutor.Execute(WithArguments( - GetBuildCommand(GenerateResult.ArtifactsPaths, BuildPartition, $"{Arguments} --no-restore --no-dependencies", "build-no-restore-no-deps"))); - public DotNetCliCommandResult Publish() => DotNetCliCommandExecutor.Execute(WithArguments( GetPublishCommand(GenerateResult.ArtifactsPaths, BuildPartition, Arguments, "publish"))); - public DotNetCliCommandResult PublishNoBuildAndNoRestore() + // PublishNoBuildAndNoRestore was removed because we set --output in the build step. We use the implicit build included in the publish command. + public DotNetCliCommandResult PublishNoRestore() => DotNetCliCommandExecutor.Execute(WithArguments( - GetPublishCommand(GenerateResult.ArtifactsPaths, BuildPartition, $"{Arguments} --no-build --no-restore", "publish-no-build-no-restore"))); + GetPublishCommand(GenerateResult.ArtifactsPaths, BuildPartition, $"{Arguments} --no-restore", "publish-no-restore"))); + [Obsolete] internal static IEnumerable GetAddPackagesCommands(BuildPartition buildPartition) => GetNuGetAddPackageCommands(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver); - internal static string GetRestoreCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string? extraArguments = null, string? binLogSuffix = null) + internal static string GetRestoreCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string? extraArguments = null, string? binLogSuffix = null, bool excludeOutput = false) => new StringBuilder() .AppendArgument("restore") .AppendArgument(string.IsNullOrEmpty(artifactsPaths.PackagesDirectoryName) ? string.Empty : $"--packages \"{artifactsPaths.PackagesDirectoryName}\"") @@ -174,9 +169,10 @@ internal static string GetRestoreCommand(ArtifactsPaths artifactsPaths, BuildPar .AppendArgument(extraArguments) .AppendArgument(GetMandatoryMsBuildSettings(buildPartition.BuildConfiguration)) .AppendArgument(GetMsBuildBinLogArgument(buildPartition, binLogSuffix)) + .MaybeAppendOutputPaths(artifactsPaths, true, excludeOutput) .ToString(); - internal static string GetBuildCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string? extraArguments = null, string? binLogSuffix = null) + internal static string GetBuildCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string? extraArguments = null, string? binLogSuffix = null, bool excludeOutput = false) => new StringBuilder() .AppendArgument($"build -c {buildPartition.BuildConfiguration}") // we don't need to specify TFM, our auto-generated project contains always single one .AppendArgument(GetCustomMsBuildArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)) @@ -184,6 +180,7 @@ internal static string GetBuildCommand(ArtifactsPaths artifactsPaths, BuildParti .AppendArgument(GetMandatoryMsBuildSettings(buildPartition.BuildConfiguration)) .AppendArgument(string.IsNullOrEmpty(artifactsPaths.PackagesDirectoryName) ? string.Empty : $"/p:NuGetPackageRoot=\"{artifactsPaths.PackagesDirectoryName}\"") .AppendArgument(GetMsBuildBinLogArgument(buildPartition, binLogSuffix)) + .MaybeAppendOutputPaths(artifactsPaths, excludeOutput: excludeOutput) .ToString(); internal static string GetPublishCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string? extraArguments = null, string? binLogSuffix = null) @@ -194,7 +191,7 @@ internal static string GetPublishCommand(ArtifactsPaths artifactsPaths, BuildPar .AppendArgument(GetMandatoryMsBuildSettings(buildPartition.BuildConfiguration)) .AppendArgument(string.IsNullOrEmpty(artifactsPaths.PackagesDirectoryName) ? string.Empty : $"/p:NuGetPackageRoot=\"{artifactsPaths.PackagesDirectoryName}\"") .AppendArgument(GetMsBuildBinLogArgument(buildPartition, binLogSuffix)) - .AppendArgument($"--output \"{artifactsPaths.BinariesDirectoryPath}\"") + .MaybeAppendOutputPaths(artifactsPaths) .ToString(); private static string GetMsBuildBinLogArgument(BuildPartition buildPartition, string suffix) @@ -215,6 +212,7 @@ private static string GetCustomMsBuildArguments(BenchmarkCase benchmarkCase, IRe return string.Join(" ", msBuildArguments.Select(arg => arg.TextRepresentation)); } + [Obsolete] private static IEnumerable GetNuGetAddPackageCommands(BenchmarkCase benchmarkCase, IResolver resolver) { if (!benchmarkCase.Job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic)) @@ -229,7 +227,7 @@ private static string GetMandatoryMsBuildSettings(string buildConfiguration) { // we use these settings to make sure that MSBuild does the job and simply quits without spawning any long living processes // we want to avoid "file in use" and "zombie processes" issues - const string NoMsBuildZombieProcesses = "/p:UseSharedCompilation=false /p:BuildInParallel=false /m:1 /p:Deterministic=true"; + const string NoMsBuildZombieProcesses = "--nodeReuse:false /p:UseSharedCompilation=false /p:Deterministic=true"; const string EnforceOptimizations = "/p:Optimize=true"; if (string.Equals(buildConfiguration, RuntimeInformation.DebugConfigurationName, StringComparison.OrdinalIgnoreCase)) @@ -240,6 +238,7 @@ private static string GetMandatoryMsBuildSettings(string buildConfiguration) return $"{NoMsBuildZombieProcesses} {EnforceOptimizations}"; } + [Obsolete] private static string BuildAddPackageCommand(NuGetReference reference) { var commandBuilder = new StringBuilder(); @@ -262,4 +261,23 @@ private static string BuildAddPackageCommand(NuGetReference reference) return commandBuilder.ToString(); } } + + internal static class DotNetCliCommandExtensions + { + // Fix #1377 (see comments in #1773). + // We force the project to output binaries to a new directory. + // Specifying --output and --no-dependencies breaks the build (because the previous build was not done using the custom output path), + // so we don't include it if we're building no-deps (only supported for integration tests). + internal static StringBuilder MaybeAppendOutputPaths(this StringBuilder stringBuilder, ArtifactsPaths artifactsPaths, bool isRestore = false, bool excludeOutput = false) + => excludeOutput + ? stringBuilder + : stringBuilder + // Use AltDirectorySeparatorChar so it's not interpreted as an escaped quote `\"`. + .AppendArgument($"/p:ArtifactsPath=\"{artifactsPaths.BuildArtifactsDirectoryPath}{Path.AltDirectorySeparatorChar}\"") + .AppendArgument($"/p:OutDir=\"{artifactsPaths.BinariesDirectoryPath}{Path.AltDirectorySeparatorChar}\"") + // OutputPath is legacy, per-project version of OutDir. We set both just in case. https://github.com/dotnet/msbuild/issues/87 + .AppendArgument($"/p:OutputPath=\"{artifactsPaths.BinariesDirectoryPath}{Path.AltDirectorySeparatorChar}\"") + .AppendArgument($"/p:PublishDir=\"{artifactsPaths.PublishDirectoryPath}{Path.AltDirectorySeparatorChar}\"") + .AppendArgument(isRestore ? string.Empty : $"--output \"{artifactsPaths.BinariesDirectoryPath}{Path.AltDirectorySeparatorChar}\""); + } } diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs index 3532fa1e07..10df4af9b3 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; @@ -54,9 +55,9 @@ public static DotNetCliCommandResult Execute(DotNetCliCommand parameters) } } - internal static string GetDotNetSdkVersion() + internal static string? GetDotNetSdkVersion() { - using (var process = new Process { StartInfo = BuildStartInfo(customDotNetCliPath: null, workingDirectory: string.Empty, arguments: "--version") }) + using (var process = new Process { StartInfo = BuildStartInfo(customDotNetCliPath: null, workingDirectory: string.Empty, arguments: "--version", redirectStandardError: false) }) using (new ConsoleExitHandler(process, NullLogger.Instance)) { try @@ -98,7 +99,7 @@ internal static void LogEnvVars(DotNetCliCommand command) } } - internal static ProcessStartInfo BuildStartInfo(string customDotNetCliPath, string workingDirectory, string arguments, + internal static ProcessStartInfo BuildStartInfo(string? customDotNetCliPath, string workingDirectory, string arguments, IReadOnlyList? environmentVariables = null, bool redirectStandardInput = false, bool redirectStandardError = true, bool redirectStandardOutput = true) { const string dotnetMultiLevelLookupEnvVarName = "DOTNET_MULTILEVEL_LOOKUP"; @@ -137,7 +138,7 @@ internal static ProcessStartInfo BuildStartInfo(string customDotNetCliPath, stri private static string GetDefaultDotNetCliPath() { - if (!Portability.RuntimeInformation.IsLinux()) + if (!OsDetector.IsLinux()) return "dotnet"; using (var parentProcess = Process.GetProcessById(libc.getppid())) @@ -156,7 +157,7 @@ private static string GetDefaultDotNetCliPath() internal static string GetSdkPath(string cliPath) { - DotNetCliCommand cliCommand = new ( + DotNetCliCommand cliCommand = new( cliPath: cliPath, arguments: "--info", generateResult: null, diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandResult.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandResult.cs index 3d65270621..4a5b00a1fd 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandResult.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandResult.cs @@ -35,7 +35,7 @@ public static DotNetCliCommandResult Failure(TimeSpan time, string standardError [PublicAPI] public BuildResult ToBuildResult(GenerateResult generateResult) - => IsSuccess || File.Exists(generateResult.ArtifactsPaths.ExecutablePath) // dotnet cli could have successfully built the program, but returned 1 as exit code because it had some warnings + => IsSuccess ? BuildResult.Success(generateResult) : BuildResult.Failure(generateResult, AllInformation); } diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs index 55815ed005..d0595fe166 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs @@ -75,17 +75,20 @@ private ExecuteResult Execute(BenchmarkCase benchmarkCase, startInfo.SetEnvironmentVariables(benchmarkCase, resolver); - using (Process process = new () { StartInfo = startInfo }) - using (ConsoleExitHandler consoleExitHandler = new (process, logger)) - using (AsyncProcessOutputReader processOutputReader = new (process, logOutput: true, logger, readStandardError: false)) + using (Process process = new() { StartInfo = startInfo }) + using (ConsoleExitHandler consoleExitHandler = new(process, logger)) + using (AsyncProcessOutputReader processOutputReader = new(process, logOutput: true, logger, readStandardError: false)) { - Broker broker = new (logger, process, diagnoser, benchmarkCase, benchmarkId, inputFromBenchmark, acknowledgments); + Broker broker = new(logger, process, diagnoser, benchmarkCase, benchmarkId, inputFromBenchmark, acknowledgments); logger.WriteLineInfo($"// Execute: {process.StartInfo.FileName} {process.StartInfo.Arguments} in {process.StartInfo.WorkingDirectory}"); - diagnoser?.Handle(HostSignal.BeforeProcessStart, new DiagnoserActionParameters(process, benchmarkCase, benchmarkId)); + diagnoser?.Handle(HostSignal.BeforeProcessStart, broker.DiagnoserActionParameters); process.Start(); + + diagnoser?.Handle(HostSignal.AfterProcessStart, broker.DiagnoserActionParameters); + processOutputReader.BeginRead(); process.EnsureHighPriority(logger); diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs index a432fbf881..aaa2cda154 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs @@ -12,6 +12,8 @@ public abstract class DotNetCliGenerator : GeneratorBase { private static readonly string[] ProjectExtensions = { ".csproj", ".fsproj", ".vbroj" }; + private static readonly string[] SolutionExtensions = { ".sln", ".slnx" }; + [PublicAPI] public string TargetFrameworkMoniker { get; } [PublicAPI] public string CliPath { get; } @@ -111,7 +113,7 @@ protected override void GenerateBuildScript(BuildPartition buildPartition, Artif private static bool IsRootSolutionFolder(DirectoryInfo directoryInfo) => directoryInfo .GetFileSystemInfos() - .Any(fileInfo => fileInfo.Extension == ".sln" || fileInfo.Name == "global.json"); + .Any(fileInfo => SolutionExtensions.Contains(fileInfo.Extension) || fileInfo.Name == "global.json"); private static bool IsRootProjectFolder(DirectoryInfo directoryInfo) => directoryInfo diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliPublisher.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliPublisher.cs index 960558d5c0..83158e0b1e 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliPublisher.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliPublisher.cs @@ -8,18 +8,21 @@ namespace BenchmarkDotNet.Toolchains.DotNetCli { public class DotNetCliPublisher : IBuilder { - public DotNetCliPublisher(string? customDotNetCliPath = null, string? extraArguments = null, IReadOnlyList environmentVariables = null) + public DotNetCliPublisher( + string? customDotNetCliPath = null, + string? extraArguments = null, + IReadOnlyList? environmentVariables = null) { CustomDotNetCliPath = customDotNetCliPath; ExtraArguments = extraArguments; EnvironmentVariables = environmentVariables; } - private string CustomDotNetCliPath { get; } + private string? CustomDotNetCliPath { get; } - private string ExtraArguments { get; } + private string? ExtraArguments { get; } - private IReadOnlyList EnvironmentVariables { get; } + private IReadOnlyList? EnvironmentVariables { get; } public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger) => new DotNetCliCommand( diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/MsBuildErrorMapper.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/MsBuildErrorMapper.cs index bd5fba4123..9c95be192f 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/MsBuildErrorMapper.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/MsBuildErrorMapper.cs @@ -69,6 +69,8 @@ private static string Map(Capture capture) return "net472"; case ".NETFramework,Version=v4.8": return "net48"; + case ".NETFramework,Version=v4.8.1": + return "net481"; case ".NETCoreApp,Version=v2.0": return "netcoreapp2.0"; case ".NETCoreApp,Version=v2.1": @@ -83,6 +85,14 @@ private static string Map(Capture capture) return "net5.0"; case ".NETCoreApp,Version=v6.0": return "net6.0"; + case ".NETCoreApp,Version=v7.0": + return "net7.0"; + case ".NETCoreApp,Version=v8.0": + return "net8.0"; + case ".NETCoreApp,Version=v9.0": + return "net9.0"; + case ".NETCoreApp,Version=v10.0": + return "net10.0"; default: return capture.Value; // we don't want to throw for future versions of .NET } diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs index 664d1333f1..807e394682 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs @@ -9,20 +9,21 @@ namespace BenchmarkDotNet.Toolchains.DotNetCli [PublicAPI] public class NetCoreAppSettings { - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp20 = new ("netcoreapp2.0", null, ".NET Core 2.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp21 = new ("netcoreapp2.1", null, ".NET Core 2.1"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp22 = new ("netcoreapp2.2", null, ".NET Core 2.2"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp30 = new ("netcoreapp3.0", null, ".NET Core 3.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp31 = new ("netcoreapp3.1", null, ".NET Core 3.1"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp50 = new ("net5.0", null, ".NET 5.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp60 = new ("net6.0", null, ".NET 6.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp70 = new ("net7.0", null, ".NET 7.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp80 = new ("net8.0", null, ".NET 8.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp90 = new ("net9.0", null, ".NET 9.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp20 = new("netcoreapp2.0", null, ".NET Core 2.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp21 = new("netcoreapp2.1", null, ".NET Core 2.1"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp22 = new("netcoreapp2.2", null, ".NET Core 2.2"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp30 = new("netcoreapp3.0", null, ".NET Core 3.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp31 = new("netcoreapp3.1", null, ".NET Core 3.1"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp50 = new("net5.0", null, ".NET 5.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp60 = new("net6.0", null, ".NET 6.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp70 = new("net7.0", null, ".NET 7.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp80 = new("net8.0", null, ".NET 8.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp90 = new("net9.0", null, ".NET 9.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp10_0 = new("net10.0", null, ".NET 10.0"); /// /// - /// sample values: netcoreapp2.0, netcoreapp2.1 + /// sample values: net6.0, net8.0 /// /// /// used in the auto-generated .csproj file @@ -43,7 +44,7 @@ public class NetCoreAppSettings [PublicAPI] public NetCoreAppSettings( string targetFrameworkMoniker, - string runtimeFrameworkVersion, + string? runtimeFrameworkVersion, string name, string? customDotNetCliPath = null, string? packagesPath = null, @@ -64,33 +65,33 @@ public NetCoreAppSettings( } /// - /// sample values: netcoreapp2.0, netcoreapp2.1 + /// sample values: net6.0, net8.0 /// public string TargetFrameworkMoniker { get; } - public string RuntimeFrameworkVersion { get; } + public string? RuntimeFrameworkVersion { get; } /// /// display name used for showing the results /// public string Name { get; } - public string CustomDotNetCliPath { get; } + public string? CustomDotNetCliPath { get; } /// /// The directory to restore packages to. /// - public string PackagesPath { get; } + public string? PackagesPath { get; } /// /// Path to a custom runtime pack. /// - public string CustomRuntimePack { get; } + public string? CustomRuntimePack { get; } /// /// Path to the Mono AOT Compiler /// - public string AOTCompilerPath { get; } + public string? AOTCompilerPath { get; } /// /// Mono AOT Compiler mode, either 'mini' or 'llvm' diff --git a/src/BenchmarkDotNet/Toolchains/Executor.cs b/src/BenchmarkDotNet/Toolchains/Executor.cs index 735c6f7437..adc05e12e6 100644 --- a/src/BenchmarkDotNet/Toolchains/Executor.cs +++ b/src/BenchmarkDotNet/Toolchains/Executor.cs @@ -41,16 +41,16 @@ private static ExecuteResult Execute(BenchmarkCase benchmarkCase, BenchmarkId be { try { - using AnonymousPipeServerStream inputFromBenchmark = new (PipeDirection.In, HandleInheritability.Inheritable); - using AnonymousPipeServerStream acknowledgments = new (PipeDirection.Out, HandleInheritability.Inheritable); + using AnonymousPipeServerStream inputFromBenchmark = new(PipeDirection.In, HandleInheritability.Inheritable); + using AnonymousPipeServerStream acknowledgments = new(PipeDirection.Out, HandleInheritability.Inheritable); string args = benchmarkId.ToArguments(inputFromBenchmark.GetClientHandleAsString(), acknowledgments.GetClientHandleAsString()); - using (Process process = new () { StartInfo = CreateStartInfo(benchmarkCase, artifactsPaths, args, resolver) }) - using (ConsoleExitHandler consoleExitHandler = new (process, logger)) - using (AsyncProcessOutputReader processOutputReader = new (process, logOutput: true, logger, readStandardError: false)) + using (Process process = new() { StartInfo = CreateStartInfo(benchmarkCase, artifactsPaths, args, resolver) }) + using (ConsoleExitHandler consoleExitHandler = new(process, logger)) + using (AsyncProcessOutputReader processOutputReader = new(process, logOutput: true, logger, readStandardError: false)) { - Broker broker = new (logger, process, diagnoser, benchmarkCase, benchmarkId, inputFromBenchmark, acknowledgments); + Broker broker = new(logger, process, diagnoser, benchmarkCase, benchmarkId, inputFromBenchmark, acknowledgments); diagnoser?.Handle(HostSignal.BeforeProcessStart, new DiagnoserActionParameters(process, benchmarkCase, benchmarkId)); @@ -79,6 +79,8 @@ private static ExecuteResult Execute(Process process, BenchmarkCase benchmarkCas return new ExecuteResult(true, null, null, Array.Empty(), Array.Empty(), Array.Empty(), launchIndex); } + broker.Diagnoser?.Handle(HostSignal.AfterProcessStart, broker.DiagnoserActionParameters); + processOutputReader.BeginRead(); process.EnsureHighPriority(logger); @@ -146,7 +148,7 @@ private static ProcessStartInfo CreateStartInfo(BenchmarkCase benchmarkCase, Art case MonoAotLLVMRuntime _: start.FileName = exePath; start.Arguments = args; - start.WorkingDirectory = artifactsPaths.BinariesDirectoryPath; + start.WorkingDirectory = Path.Combine(artifactsPaths.BinariesDirectoryPath, "publish"); break; case CustomRuntime _: start.FileName = exePath; diff --git a/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs b/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs index 0d7ab30d7d..165dedcbba 100644 --- a/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs +++ b/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs @@ -2,6 +2,8 @@ using System.IO; using System.Text; using BenchmarkDotNet.Code; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; @@ -43,16 +45,23 @@ public GenerateResult GenerateProject(BuildPartition buildPartition, ILogger log [PublicAPI] protected abstract string GetBuildArtifactsDirectoryPath(BuildPartition assemblyLocation, string programName); /// - /// returns a path where executable should be found after the build + /// returns a path where executable should be found after the build (usually \bin) /// [PublicAPI] protected virtual string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) => buildArtifactsDirectoryPath; + /// + /// returns a path where the publish directory should be found after the build (usually \publish) + /// + [PublicAPI] + protected virtual string GetPublishDirectoryPath(string buildArtifactsDirectoryPath, string configuration) + => Path.Combine(buildArtifactsDirectoryPath, "publish"); + /// /// returns OS-specific executable extension /// [PublicAPI] protected virtual string GetExecutableExtension() - => RuntimeInformation.ExecutableExtension; + => OsDetector.ExecutableExtension; /// /// returns a path to the auto-generated .csproj file @@ -96,6 +105,7 @@ [PublicAPI] protected virtual void GenerateProject(BuildPartition buildPartition [PublicAPI] protected virtual void GenerateAppConfig(BuildPartition buildPartition, ArtifactsPaths artifactsPaths) { string sourcePath = buildPartition.AssemblyLocation + ".config"; + artifactsPaths.AppConfigPath.EnsureFolderExists(); using (var source = File.Exists(sourcePath) ? new StreamReader(File.OpenRead(sourcePath)) : TextReader.Null) using (var destination = new StreamWriter(File.Create(artifactsPaths.AppConfigPath), Encoding.UTF8)) @@ -128,11 +138,12 @@ private ArtifactsPaths GetArtifactsPaths(BuildPartition buildPartition, string r rootArtifactsFolderPath: rootArtifactsFolderPath, buildArtifactsDirectoryPath: buildArtifactsDirectoryPath, binariesDirectoryPath: binariesDirectoryPath, + publishDirectoryPath: GetPublishDirectoryPath(buildArtifactsDirectoryPath, buildPartition.BuildConfiguration), programCodePath: Path.Combine(buildArtifactsDirectoryPath, $"{programName}{codeFileExtension}"), appConfigPath: $"{executablePath}.config", nuGetConfigPath: Path.Combine(buildArtifactsDirectoryPath, "NuGet.config"), projectFilePath: GetProjectFilePath(buildArtifactsDirectoryPath), - buildScriptFilePath: Path.Combine(buildArtifactsDirectoryPath, $"{programName}{RuntimeInformation.ScriptFileExtension}"), + buildScriptFilePath: Path.Combine(buildArtifactsDirectoryPath, $"{programName}{OsDetector.ScriptFileExtension}"), executablePath: executablePath, programName: programName, packagesDirectoryName: GetPackagesDirectoryPath(buildArtifactsDirectoryPath)); diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/ConsumableTypeInfo.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/ConsumableTypeInfo.cs index f8d344b1c2..147a514058 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/ConsumableTypeInfo.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/ConsumableTypeInfo.cs @@ -1,5 +1,7 @@ using BenchmarkDotNet.Engines; using System; +using System.Collections.Generic; +using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; using System.Threading.Tasks; @@ -16,28 +18,24 @@ public ConsumableTypeInfo(Type methodReturnType) OriginMethodReturnType = methodReturnType; - // Please note this code does not support await over extension methods. - var getAwaiterMethod = methodReturnType.GetMethod(nameof(Task.GetAwaiter), BindingFlagsPublicInstance); - if (getAwaiterMethod == null) + // Only support (Value)Task for parity with other toolchains (and so we can use AwaitHelper). + IsAwaitable = methodReturnType == typeof(Task) || methodReturnType == typeof(ValueTask) + || (methodReturnType.GetTypeInfo().IsGenericType + && (methodReturnType.GetTypeInfo().GetGenericTypeDefinition() == typeof(Task<>) + || methodReturnType.GetTypeInfo().GetGenericTypeDefinition() == typeof(ValueTask<>))); + + if (!IsAwaitable) { WorkloadMethodReturnType = methodReturnType; } else { - var getResultMethod = getAwaiterMethod + WorkloadMethodReturnType = methodReturnType + .GetMethod(nameof(Task.GetAwaiter), BindingFlagsPublicInstance) .ReturnType - .GetMethod(nameof(TaskAwaiter.GetResult), BindingFlagsPublicInstance); - - if (getResultMethod == null) - { - WorkloadMethodReturnType = methodReturnType; - } - else - { - WorkloadMethodReturnType = getResultMethod.ReturnType; - GetAwaiterMethod = getAwaiterMethod; - GetResultMethod = getResultMethod; - } + .GetMethod(nameof(TaskAwaiter.GetResult), BindingFlagsPublicInstance) + .ReturnType; + GetResultMethod = Helpers.AwaitHelper.GetGetResultMethod(methodReturnType); } if (WorkloadMethodReturnType == null) @@ -74,7 +72,6 @@ public ConsumableTypeInfo(Type methodReturnType) public Type WorkloadMethodReturnType { get; } public Type OverheadMethodReturnType { get; } - public MethodInfo? GetAwaiterMethod { get; } public MethodInfo? GetResultMethod { get; } public bool IsVoid { get; } @@ -82,6 +79,6 @@ public ConsumableTypeInfo(Type methodReturnType) public bool IsConsumable { get; } public FieldInfo? WorkloadConsumableField { get; } - public bool IsAwaitable => GetAwaiterMethod != null && GetResultMethod != null; + public bool IsAwaitable { get; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs index 9048474329..4796c84a77 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs @@ -247,6 +247,10 @@ private static void EmitNoArgsMethodCallPopReturn( private TypeBuilder runnableBuilder; private ConsumableTypeInfo consumableInfo; private ConsumeEmitter consumeEmitter; + private ConsumableTypeInfo globalSetupReturnInfo; + private ConsumableTypeInfo globalCleanupReturnInfo; + private ConsumableTypeInfo iterationSetupReturnInfo; + private ConsumableTypeInfo iterationCleanupReturnInfo; private FieldBuilder globalSetupActionField; private FieldBuilder globalCleanupActionField; @@ -358,6 +362,10 @@ private void InitForEmitRunnable(BenchmarkBuildInfo newBenchmark) consumableInfo = new ConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.WorkloadMethod.ReturnType); consumeEmitter = ConsumeEmitter.GetConsumeEmitter(consumableInfo); + globalSetupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.GlobalSetupMethod?.ReturnType); + globalCleanupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.GlobalCleanupMethod?.ReturnType); + iterationSetupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.IterationSetupMethod?.ReturnType); + iterationCleanupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.IterationCleanupMethod?.ReturnType); // Init types runnableBuilder = DefineRunnableTypeBuilder(benchmark, moduleBuilder); @@ -365,6 +373,11 @@ private void InitForEmitRunnable(BenchmarkBuildInfo newBenchmark) workloadDelegateType = EmitWorkloadDelegateType(); } + private static ConsumableTypeInfo GetConsumableTypeInfo(Type methodReturnType) + { + return methodReturnType == null ? null : new ConsumableTypeInfo(methodReturnType); + } + private Type EmitOverheadDelegateType() { // .class public auto ansi sealed BenchmarkDotNet.Autogenerated.Runnable_0OverheadDelegate @@ -434,7 +447,7 @@ private void DefineFields() Type argLocalsType; Type argFieldType; - MethodInfo? opConversion = null; + MethodInfo opConversion = null; if (parameterType.IsByRef) { argLocalsType = parameterType; @@ -582,42 +595,28 @@ private MethodInfo EmitWorkloadImplementation(string methodName) workloadInvokeMethod.ReturnParameter, args); args = methodBuilder.GetEmitParameters(args); - var callResultType = consumableInfo.OriginMethodReturnType; - var awaiterType = consumableInfo.GetAwaiterMethod?.ReturnType - ?? throw new InvalidOperationException($"Bug: {nameof(consumableInfo.GetAwaiterMethod)} is null"); var ilBuilder = methodBuilder.GetILGenerator(); /* - .locals init ( - [0] valuetype [mscorlib]System.Runtime.CompilerServices.TaskAwaiter`1 - ) - */ - var callResultLocal = - ilBuilder.DeclareOptionalLocalForInstanceCall(callResultType, consumableInfo.GetAwaiterMethod); - var awaiterLocal = - ilBuilder.DeclareOptionalLocalForInstanceCall(awaiterType, consumableInfo.GetResultMethod); - - /* - // return TaskSample(arg0). ... ; - IL_0000: ldarg.0 - IL_0001: ldarg.1 - IL_0002: call instance class [mscorlib]System.Threading.Tasks.Task`1 [BenchmarkDotNet]BenchmarkDotNet.Samples.SampleBenchmark::TaskSample(int64) - */ + IL_0026: ldarg.0 + IL_0027: ldloc.0 + IL_0028: ldloc.1 + IL_0029: ldloc.2 + IL_002a: ldloc.3 + IL_002b: call instance class [System.Private.CoreLib]System.Threading.Tasks.Task`1 BenchmarkDotNet.Helpers.Runnable_0::WorkloadMethod(string, string, string, string) + */ if (!Descriptor.WorkloadMethod.IsStatic) ilBuilder.Emit(OpCodes.Ldarg_0); ilBuilder.EmitLdargs(args); ilBuilder.Emit(OpCodes.Call, Descriptor.WorkloadMethod); /* - // ... .GetAwaiter().GetResult(); - IL_0007: callvirt instance valuetype [mscorlib]System.Runtime.CompilerServices.TaskAwaiter`1 class [mscorlib]System.Threading.Tasks.Task`1::GetAwaiter() - IL_000c: stloc.0 - IL_000d: ldloca.s 0 - IL_000f: call instance !0 valuetype [mscorlib]System.Runtime.CompilerServices.TaskAwaiter`1::GetResult() - */ - ilBuilder.EmitInstanceCallThisValueOnStack(callResultLocal, consumableInfo.GetAwaiterMethod); - ilBuilder.EmitInstanceCallThisValueOnStack(awaiterLocal, consumableInfo.GetResultMethod); + // BenchmarkDotNet.Helpers.AwaitHelper.GetResult(...); + IL_000e: call !!0 BenchmarkDotNet.Helpers.AwaitHelper::GetResult(valuetype [System.Runtime]System.Threading.Tasks.ValueTask`1) + */ + + ilBuilder.Emit(OpCodes.Call, consumableInfo.GetResultMethod); /* IL_0014: ret @@ -833,19 +832,6 @@ .locals init ( var skipFirstArg = workloadMethod.IsStatic; var argLocals = EmitDeclareArgLocals(ilBuilder, skipFirstArg); - LocalBuilder? callResultLocal = null; - LocalBuilder? awaiterLocal = null; - if (consumableInfo.IsAwaitable) - { - var callResultType = consumableInfo.OriginMethodReturnType; - var awaiterType = consumableInfo.GetAwaiterMethod?.ReturnType - ?? throw new InvalidOperationException($"Bug: {nameof(consumableInfo.GetAwaiterMethod)} is null"); - callResultLocal = - ilBuilder.DeclareOptionalLocalForInstanceCall(callResultType, consumableInfo.GetAwaiterMethod); - awaiterLocal = - ilBuilder.DeclareOptionalLocalForInstanceCall(awaiterType, consumableInfo.GetResultMethod); - } - consumeEmitter.DeclareDisassemblyDiagnoserLocals(ilBuilder); var notElevenLabel = ilBuilder.DefineLabel(); @@ -870,29 +856,27 @@ .locals init ( EmitLoadArgFieldsToLocals(ilBuilder, argLocals, skipFirstArg); /* - // return TaskSample(_argField) ... ; - IL_0011: ldarg.0 - IL_0012: ldloc.0 - IL_0013: call instance class [mscorlib]System.Threading.Tasks.Task`1 [BenchmarkDotNet]BenchmarkDotNet.Samples.SampleBenchmark::TaskSample(int64) - IL_0018: ret + IL_0026: ldarg.0 + IL_0027: ldloc.0 + IL_0028: ldloc.1 + IL_0029: ldloc.2 + IL_002a: ldloc.3 + IL_002b: call instance class [System.Private.CoreLib]System.Threading.Tasks.Task`1 BenchmarkDotNet.Helpers.Runnable_0::WorkloadMethod(string, string, string, string) */ - if (!workloadMethod.IsStatic) + { ilBuilder.Emit(OpCodes.Ldarg_0); + } ilBuilder.EmitLdLocals(argLocals); ilBuilder.Emit(OpCodes.Call, workloadMethod); if (consumableInfo.IsAwaitable) { /* - // ... .GetAwaiter().GetResult(); - IL_0007: callvirt instance valuetype [mscorlib]System.Runtime.CompilerServices.TaskAwaiter`1 class [mscorlib]System.Threading.Tasks.Task`1::GetAwaiter() - IL_000c: stloc.0 - IL_000d: ldloca.s 0 - IL_000f: call instance !0 valuetype [mscorlib]System.Runtime.CompilerServices.TaskAwaiter`1::GetResult() - */ - ilBuilder.EmitInstanceCallThisValueOnStack(callResultLocal, consumableInfo.GetAwaiterMethod); - ilBuilder.EmitInstanceCallThisValueOnStack(awaiterLocal, consumableInfo.GetResultMethod); + // BenchmarkDotNet.Helpers.AwaitHelper.GetResult(...); + IL_000e: call !!0 BenchmarkDotNet.Helpers.AwaitHelper::GetResult(valuetype [System.Runtime]System.Threading.Tasks.ValueTask`1) + */ + ilBuilder.Emit(OpCodes.Call, consumableInfo.GetResultMethod); } /* @@ -919,34 +903,84 @@ private void EmitSetupCleanupMethods() { // Emit Setup/Cleanup methods // We emit empty method instead of EmptyAction = "() => { }" - globalSetupMethod = EmitWrapperMethod( - GlobalSetupMethodName, - Descriptor.GlobalSetupMethod); - globalCleanupMethod = EmitWrapperMethod( - GlobalCleanupMethodName, - Descriptor.GlobalCleanupMethod); - iterationSetupMethod = EmitWrapperMethod( - IterationSetupMethodName, - Descriptor.IterationSetupMethod); - iterationCleanupMethod = EmitWrapperMethod( - IterationCleanupMethodName, - Descriptor.IterationCleanupMethod); + globalSetupMethod = EmitWrapperMethod(GlobalSetupMethodName, Descriptor.GlobalSetupMethod, globalSetupReturnInfo); + globalCleanupMethod = EmitWrapperMethod(GlobalCleanupMethodName, Descriptor.GlobalCleanupMethod, globalCleanupReturnInfo); + iterationSetupMethod = EmitWrapperMethod(IterationSetupMethodName, Descriptor.IterationSetupMethod, iterationSetupReturnInfo); + iterationCleanupMethod = EmitWrapperMethod(IterationCleanupMethodName, Descriptor.IterationCleanupMethod, iterationCleanupReturnInfo); } - private MethodBuilder EmitWrapperMethod(string methodName, MethodInfo optionalTargetMethod) + private MethodBuilder EmitWrapperMethod(string methodName, MethodInfo optionalTargetMethod, ConsumableTypeInfo returnTypeInfo) { var methodBuilder = runnableBuilder.DefinePrivateVoidInstanceMethod(methodName); var ilBuilder = methodBuilder.GetILGenerator(); if (optionalTargetMethod != null) - EmitNoArgsMethodCallPopReturn(methodBuilder, optionalTargetMethod, ilBuilder, forceDirectCall: true); + { + if (returnTypeInfo?.IsAwaitable == true) + { + EmitAwaitableSetupTeardown(methodBuilder, optionalTargetMethod, ilBuilder, returnTypeInfo); + } + else + { + EmitNoArgsMethodCallPopReturn(methodBuilder, optionalTargetMethod, ilBuilder, forceDirectCall: true); + } + } ilBuilder.EmitVoidReturn(methodBuilder); return methodBuilder; } + private void EmitAwaitableSetupTeardown( + MethodBuilder methodBuilder, + MethodInfo targetMethod, + ILGenerator ilBuilder, + ConsumableTypeInfo returnTypeInfo) + { + if (targetMethod == null) + throw new ArgumentNullException(nameof(targetMethod)); + + if (returnTypeInfo.WorkloadMethodReturnType == typeof(void)) + { + ilBuilder.Emit(OpCodes.Ldarg_0); + } + /* + // call for instance + // GlobalSetup(); + IL_0006: ldarg.0 + IL_0007: call instance void [BenchmarkDotNet]BenchmarkDotNet.Samples.SampleBenchmark::GlobalSetup() + */ + /* + // call for static + // GlobalSetup(); + IL_0006: call string [BenchmarkDotNet]BenchmarkDotNet.Samples.SampleBenchmark::GlobalCleanup() + */ + if (targetMethod.IsStatic) + { + ilBuilder.Emit(OpCodes.Call, targetMethod); + + } + else if (methodBuilder.IsStatic) + { + throw new InvalidOperationException( + $"[BUG] Static method {methodBuilder.Name} tries to call instance member {targetMethod.Name}"); + } + else + { + ilBuilder.Emit(OpCodes.Ldarg_0); + ilBuilder.Emit(OpCodes.Call, targetMethod); + } + + /* + // BenchmarkDotNet.Helpers.AwaitHelper.GetResult(...); + IL_000e: call !!0 BenchmarkDotNet.Helpers.AwaitHelper::GetResult(valuetype [System.Runtime]System.Threading.Tasks.ValueTask`1) + */ + + ilBuilder.Emit(OpCodes.Call, returnTypeInfo.GetResultMethod); + ilBuilder.Emit(OpCodes.Pop); + } + private void EmitCtorBody() { var ilBuilder = ctorMethod.GetILGenerator(); @@ -1012,7 +1046,7 @@ private MethodBuilder EmitRunMethod() { var prepareForRunMethodTemplate = typeof(RunnableReuse).GetMethod(nameof(RunnableReuse.PrepareForRun)) ?? throw new MissingMemberException(nameof(RunnableReuse.PrepareForRun)); - var resultTuple = new ValueTuple(); + (Job, EngineParameters, IEngineFactory) resultTuple = new(); /* .method public hidebysig static diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitArtifactsPath.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitArtifactsPath.cs index c2b3199d93..ca8ca60974 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitArtifactsPath.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitArtifactsPath.cs @@ -12,6 +12,7 @@ public InProcessEmitArtifactsPath( baseArtifacts.RootArtifactsFolderPath, baseArtifacts.BuildArtifactsDirectoryPath, baseArtifacts.BinariesDirectoryPath, + baseArtifacts.PublishDirectoryPath, baseArtifacts.ProgramCodePath, baseArtifacts.AppConfigPath, baseArtifacts.NuGetConfigPath, diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs index 64d2019a75..60d6f8a6ae 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Reflection; using System.Threading; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; @@ -59,7 +60,7 @@ public ExecuteResult Execute(ExecuteParameters executeParameters) if (executeParameters.BenchmarkCase.Descriptor.WorkloadMethod .GetCustomAttributes(false) .Any() && - Portability.RuntimeInformation.IsWindows()) + OsDetector.IsWindows()) { runThread.SetApartmentState(ApartmentState.STA); } diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitGenerator.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitGenerator.cs index c169c91903..5e74d65f12 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitGenerator.cs @@ -47,6 +47,7 @@ private ArtifactsPaths GetArtifactsPaths(BuildPartition buildPartition, string r rootArtifactsFolderPath: rootArtifactsFolderPath, buildArtifactsDirectoryPath: buildArtifactsDirectoryPath, binariesDirectoryPath: binariesDirectoryPath, + publishDirectoryPath: null, programCodePath: null, appConfigPath: null, nuGetConfigPath: null, diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/BenchmarkActionFactory_Implementations.cs b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/BenchmarkActionFactory_Implementations.cs index 551e3001c9..aee0a8f998 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/BenchmarkActionFactory_Implementations.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/BenchmarkActionFactory_Implementations.cs @@ -118,7 +118,7 @@ public BenchmarkActionTask(object instance, MethodInfo method, int unrollFactor) private void Overhead() { } // must be kept in sync with TaskDeclarationsProvider.TargetMethodDelegate - private void ExecuteBlocking() => startTaskCallback.Invoke().GetAwaiter().GetResult(); + private void ExecuteBlocking() => Helpers.AwaitHelper.GetResult(startTaskCallback.Invoke()); [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] private void WorkloadActionUnroll(long repeatCount) @@ -165,7 +165,7 @@ public BenchmarkActionTask(object instance, MethodInfo method, int unrollFactor) private T Overhead() => default; // must be kept in sync with GenericTaskDeclarationsProvider.TargetMethodDelegate - private T ExecuteBlocking() => startTaskCallback().GetAwaiter().GetResult(); + private T ExecuteBlocking() => Helpers.AwaitHelper.GetResult(startTaskCallback.Invoke()); private void InvokeSingleHardcoded() => result = callback(); @@ -217,7 +217,7 @@ public BenchmarkActionValueTask(object instance, MethodInfo method, int unrollFa private T Overhead() => default; // must be kept in sync with GenericTaskDeclarationsProvider.TargetMethodDelegate - private T ExecuteBlocking() => startTaskCallback().GetAwaiter().GetResult(); + private T ExecuteBlocking() => Helpers.AwaitHelper.GetResult(startTaskCallback.Invoke()); private void InvokeSingleHardcoded() => result = callback(); diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs index 49cd24e3d4..b50ba77c1c 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Reflection; using System.Threading; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; @@ -59,7 +60,7 @@ public ExecuteResult Execute(ExecuteParameters executeParameters) var runThread = new Thread(() => exitCode = ExecuteCore(host, executeParameters)); if (executeParameters.BenchmarkCase.Descriptor.WorkloadMethod.GetCustomAttributes(false).Any() && - Portability.RuntimeInformation.IsWindows()) + OsDetector.IsWindows()) { runThread.SetApartmentState(ApartmentState.STA); } diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs index a52e7ebcf8..18ee40c30a 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs @@ -1,4 +1,5 @@ -using System; +using System; +using System.Diagnostics.CodeAnalysis; using System.Reflection; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; @@ -15,6 +16,9 @@ namespace BenchmarkDotNet.Toolchains.InProcess.NoEmit /// internal class InProcessNoEmitRunner { +#if NET6_0_OR_GREATER + [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Runnable))] +#endif public static int Run(IHost host, BenchmarkCase benchmarkCase) { // the first thing to do is to let diagnosers hook in before anything happens @@ -157,4 +161,4 @@ public static void RunCore(IHost host, BenchmarkCase benchmarkCase) } } } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs index acbeeffe38..30135c15cd 100644 --- a/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs @@ -48,12 +48,14 @@ public override IEnumerable Validate(BenchmarkCase benchmarkCas benchmarkCase); } +#pragma warning disable CS0618 // Type or member is obsolete if (benchmarkCase.Job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic)) { yield return new ValidationError(true, "The MonoAOT toolchain does not allow specifying NuGet package dependencies", benchmarkCase); } +#pragma warning restore CS0618 // Type or member is obsolete } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs index 56f2ebaa56..222eb2111b 100644 --- a/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs @@ -12,6 +12,7 @@ public class MonoToolchain : CsProjCoreToolchain, IEquatable [PublicAPI] public static readonly IToolchain Mono70 = From(new NetCoreAppSettings("net7.0", null, "mono70")); [PublicAPI] public static readonly IToolchain Mono80 = From(new NetCoreAppSettings("net8.0", null, "mono80")); [PublicAPI] public static readonly IToolchain Mono90 = From(new NetCoreAppSettings("net9.0", null, "mono90")); + [PublicAPI] public static readonly IToolchain Mono10_0 = From(new NetCoreAppSettings("net10.0", null, "mono10_0")); private MonoToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) : base(name, generator, builder, executor, customDotNetCliPath) diff --git a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs index d17fad7d00..6ba5968617 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs @@ -1,6 +1,7 @@ using System.IO; using System.Text; using System.Xml; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; @@ -42,7 +43,6 @@ protected override void GenerateProject(BuildPartition buildPartition, Artifacts .Replace("$TFM$", TargetFrameworkMoniker) .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) .Replace("$COPIEDSETTINGS$", customProperties) - .Replace("$CONFIGURATIONNAME$", buildPartition.BuildConfiguration) .Replace("$SDKNAME$", sdkName) .Replace("$RUNTIMEPACK$", CustomRuntimePack ?? "") .Replace("$COMPILERBINARYPATH$", AotCompilerPath) @@ -53,12 +53,15 @@ protected override void GenerateProject(BuildPartition buildPartition, Artifacts File.WriteAllText(artifactsPaths.ProjectFilePath, content); } + protected override string GetPublishDirectoryPath(string buildArtifactsDirectoryPath, string configuration) + => Path.Combine(GetBinariesDirectoryPath(buildArtifactsDirectoryPath, configuration), "publish"); + protected override string GetExecutablePath(string binariesDirectoryPath, string programName) - => Portability.RuntimeInformation.IsWindows() - ? Path.Combine(binariesDirectoryPath, $"{programName}.exe") - : Path.Combine(binariesDirectoryPath, programName); + => OsDetector.IsWindows() + ? Path.Combine(binariesDirectoryPath, "publish", $"{programName}.exe") + : Path.Combine(binariesDirectoryPath, "publish", programName); protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - => Path.Combine(buildArtifactsDirectoryPath, "bin", TargetFrameworkMoniker, CustomDotNetCliToolchainBuilder.GetPortableRuntimeIdentifier(), "publish"); + => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, TargetFrameworkMoniker, CustomDotNetCliToolchainBuilder.GetPortableRuntimeIdentifier()); } } diff --git a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolChain.cs b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolChain.cs index 0d77160e06..06ef8ea8c6 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolChain.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolChain.cs @@ -1,24 +1,45 @@ +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Validators; +using System.Collections.Generic; namespace BenchmarkDotNet.Toolchains.MonoAotLLVM { public class MonoAotLLVMToolChain : Toolchain { - public MonoAotLLVMToolChain(string name, IGenerator generator, IBuilder builder, IExecutor executor) + private readonly string _customDotNetCliPath; + + public MonoAotLLVMToolChain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) : base(name, generator, builder, executor) { + _customDotNetCliPath = customDotNetCliPath; } public static IToolchain From(NetCoreAppSettings netCoreAppSettings) - => new MonoAotLLVMToolChain(netCoreAppSettings.Name, - new MonoAotLLVMGenerator(netCoreAppSettings.TargetFrameworkMoniker, - netCoreAppSettings.CustomDotNetCliPath, - netCoreAppSettings.PackagesPath, - netCoreAppSettings.CustomRuntimePack, - netCoreAppSettings.AOTCompilerPath, - netCoreAppSettings.AOTCompilerMode), - new DotNetCliBuilder(netCoreAppSettings.TargetFrameworkMoniker, - netCoreAppSettings.CustomDotNetCliPath), - new Executor()); + => new MonoAotLLVMToolChain(netCoreAppSettings.Name, + new MonoAotLLVMGenerator(netCoreAppSettings.TargetFrameworkMoniker, + netCoreAppSettings.CustomDotNetCliPath, + netCoreAppSettings.PackagesPath, + netCoreAppSettings.CustomRuntimePack, + netCoreAppSettings.AOTCompilerPath, + netCoreAppSettings.AOTCompilerMode), + new DotNetCliBuilder(netCoreAppSettings.TargetFrameworkMoniker, + netCoreAppSettings.CustomDotNetCliPath), + new Executor(), + netCoreAppSettings.CustomDotNetCliPath); + + public override IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) + { + foreach (var validationError in base.Validate(benchmarkCase, resolver)) + { + yield return validationError; + } + + foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(_customDotNetCliPath, benchmarkCase)) + { + yield return validationError; + } + } } } diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs index 024c8e4242..e6756ebcbe 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs @@ -38,8 +38,8 @@ private static ExecuteResult Execute(BenchmarkCase benchmarkCase, BenchmarkId be try { using (Process process = CreateProcess(benchmarkCase, artifactsPaths, benchmarkId.ToArguments(), resolver)) - using (ConsoleExitHandler consoleExitHandler = new (process, logger)) - using (AsyncProcessOutputReader processOutputReader = new (process, logOutput: true, logger, readStandardError: false)) + using (ConsoleExitHandler consoleExitHandler = new(process, logger)) + using (AsyncProcessOutputReader processOutputReader = new(process, logOutput: true, logger, readStandardError: false)) { diagnoser?.Handle(HostSignal.BeforeProcessStart, new DiagnoserActionParameters(process, benchmarkCase, benchmarkId)); @@ -61,7 +61,7 @@ private static Process CreateProcess(BenchmarkCase benchmarkCase, ArtifactsPaths { FileName = runtime.JavaScriptEngine, Arguments = $"{runtime.JavaScriptEngineArguments} {mainJs} -- --run {artifactsPaths.ProgramName}.dll {args} ", - WorkingDirectory = artifactsPaths.BinariesDirectoryPath, + WorkingDirectory = Path.Combine(artifactsPaths.BinariesDirectoryPath, "AppBundle"), UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardInput = false, // not supported by WASM! diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs index acc1be303a..c33642f6a9 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs @@ -57,7 +57,6 @@ protected void GenerateProjectFile(BuildPartition buildPartition, ArtifactsPaths .Replace("$TFM$", TargetFrameworkMoniker) .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) .Replace("$COPIEDSETTINGS$", customProperties) - .Replace("$CONFIGURATIONNAME$", buildPartition.BuildConfiguration) .Replace("$SDKNAME$", sdkName) .Replace("$WASMDATADIR$", runtime.WasmDataDir) .Replace("$TARGET$", CustomRuntimePack != null ? "PublishWithCustomRuntimePack" : "Publish") @@ -66,11 +65,9 @@ protected void GenerateProjectFile(BuildPartition buildPartition, ArtifactsPaths File.WriteAllText(artifactsPaths.ProjectFilePath, content); } - protected override string GetExecutablePath(string binariesDirectoryPath, string programName) => Path.Combine(binariesDirectoryPath, MainJS); + protected override string GetExecutablePath(string binariesDirectoryPath, string programName) => Path.Combine(binariesDirectoryPath, "AppBundle", MainJS); protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - { - return Path.Combine(buildArtifactsDirectoryPath, "bin", TargetFrameworkMoniker, "browser-wasm", "AppBundle"); - } + => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, TargetFrameworkMoniker, "browser-wasm"); } } diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs index 37f816bde9..71ad2ade2b 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.DotNetCli; @@ -26,15 +27,16 @@ public override IEnumerable Validate(BenchmarkCase benchmarkCas yield return validationError; } - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) { yield return new ValidationError(true, $"{nameof(WasmToolchain)} is supported only on Unix, benchmark '{benchmarkCase.DisplayInfo}' might not work correctly", benchmarkCase); } - else if (IsCliPathInvalid(CustomDotNetCliPath, benchmarkCase, out var invalidCliError)) + + foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(CustomDotNetCliPath, benchmarkCase)) { - yield return invalidCliError; + yield return validationError; } } @@ -49,8 +51,7 @@ public static IToolchain From(NetCoreAppSettings netCoreAppSettings) new DotNetCliBuilder(netCoreAppSettings.TargetFrameworkMoniker, netCoreAppSettings.CustomDotNetCliPath, // aot builds can be very slow - logOutput: netCoreAppSettings.AOTCompilerMode == MonoAotLLVM.MonoAotCompilerMode.wasm, - retryFailedBuildWithNoDeps: false), + logOutput: netCoreAppSettings.AOTCompilerMode == MonoAotLLVM.MonoAotCompilerMode.wasm), new WasmExecutor(), netCoreAppSettings.CustomDotNetCliPath); } diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs index 6fb5e5421f..e9c64a82f5 100644 --- a/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs +++ b/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs @@ -3,11 +3,15 @@ using System.IO; using System.Linq; using System.Text; +using System.Xml; +using BenchmarkDotNet.ConsoleArguments; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Detectors.Cpu; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Portability.Cpu; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.CsProj; using BenchmarkDotNet.Toolchains.DotNetCli; @@ -55,7 +59,7 @@ internal Generator(string ilCompilerVersion, private readonly string ilcOptimizationPreference; private readonly string ilcInstructionSet; - protected override string GetExecutableExtension() => RuntimeInformation.ExecutableExtension; + protected override string GetExecutableExtension() => OsDetector.ExecutableExtension; protected override string GetBuildArtifactsDirectoryPath(BuildPartition buildPartition, string programName) => useTempFolderForRestore @@ -71,7 +75,6 @@ protected override void GenerateBuildScript(BuildPartition buildPartition, Artif var content = new StringBuilder(300) .AppendLine($"call {CliPath ?? "dotnet"} {DotNetCliCommand.GetRestoreCommand(artifactsPaths, buildPartition, extraArguments)}") - .AppendLine($"call {CliPath ?? "dotnet"} {DotNetCliCommand.GetBuildCommand(artifactsPaths, buildPartition, extraArguments)}") .AppendLine($"call {CliPath ?? "dotnet"} {DotNetCliCommand.GetPublishCommand(artifactsPaths, buildPartition, extraArguments)}") .ToString(); @@ -131,14 +134,16 @@ private string GenerateProjectForNuGetBuild(BuildPartition buildPartition, Artif false true false - true + true {ilcOptimizationPreference} + {ilcOptimizationPreference} {GetTrimmingSettings()} - {ilcGenerateCompleteTypeMetadata} {ilcGenerateStackTraceData} + {ilcGenerateStackTraceData} false false false + true {GetInstructionSetSettings(buildPartition)} {GetRuntimeSettings(buildPartition.RepresentativeBenchmarkCase.Job.Environment.Gc, buildPartition.Resolver)} @@ -152,14 +157,27 @@ private string GenerateProjectForNuGetBuild(BuildPartition buildPartition, Artif {string.Join(Environment.NewLine, GetRdXmlFiles(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger).Select(file => $""))} +{GetCustomProperties(buildPartition, logger)} "; + private string GetCustomProperties(BuildPartition buildPartition, ILogger logger) + { + var projectFile = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger); + var xmlDoc = new XmlDocument(); + xmlDoc.Load(projectFile.FullName); + + (string customProperties, _) = GetSettingsThatNeedToBeCopied(xmlDoc, projectFile); + return customProperties; + } + + private string GetILCompilerPackageReference() => string.IsNullOrEmpty(ilCompilerVersion) ? "" : $@""; private string GetTrimmingSettings() => rootAllApplicationAssemblies - ? "" // use the defaults + // Use the defaults + ? "" // TrimMode is set in explicit way as for older versions it might have different default value : "linklink"; @@ -218,42 +236,97 @@ private void GenerateReflectionFile(ArtifactsPaths artifactsPaths) private string GetCurrentInstructionSet(Platform platform) => string.Join(",", GetCurrentProcessInstructionSets(platform)); - // based on https://github.com/dotnet/runtime/blob/ce61c09a5f6fc71d8f717d3fc4562f42171869a0/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs#L727 - private static IEnumerable GetCurrentProcessInstructionSets(Platform platform) + // based on https://github.com/dotnet/runtime/tree/v10.0.0-rc.1.25451.107/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt + private IEnumerable GetCurrentProcessInstructionSets(Platform platform) { + if (!ConfigParser.TryParse(TargetFrameworkMoniker, out RuntimeMoniker runtimeMoniker)) + { + throw new NotSupportedException($"Invalid TFM: '{TargetFrameworkMoniker}'"); + } + + if (platform == RuntimeInformation.GetCurrentPlatform() // "native" does not support cross-compilation (so does BDN for now) + && runtimeMoniker >= RuntimeMoniker.NativeAot80) + { + yield return "native"; // added in .NET 8 https://github.com/dotnet/runtime/pull/87865 + yield break; + } + switch (platform) { case Platform.X86: case Platform.X64: if (HardwareIntrinsics.IsX86BaseSupported) yield return "base"; - if (HardwareIntrinsics.IsX86SseSupported) yield return "sse"; - if (HardwareIntrinsics.IsX86Sse2Supported) yield return "sse2"; - if (HardwareIntrinsics.IsX86Sse3Supported) yield return "sse3"; - if (HardwareIntrinsics.IsX86Ssse3Supported) yield return "ssse3"; - if (HardwareIntrinsics.IsX86Sse41Supported) yield return "sse4.1"; - if (HardwareIntrinsics.IsX86Sse42Supported) yield return "sse4.2"; + if (HardwareIntrinsics.IsX86Sse42Supported) + { + if (runtimeMoniker <= RuntimeMoniker.NativeAot10_0) yield return "sse4.2"; + if (runtimeMoniker <= RuntimeMoniker.NativeAot90) yield return "popcnt"; + } if (HardwareIntrinsics.IsX86AvxSupported) yield return "avx"; - if (HardwareIntrinsics.IsX86Avx2Supported) yield return "avx2"; - if (HardwareIntrinsics.IsX86AesSupported) yield return "aes"; - if (HardwareIntrinsics.IsX86Bmi1Supported) yield return "bmi"; - if (HardwareIntrinsics.IsX86Bmi2Supported) yield return "bmi2"; - if (HardwareIntrinsics.IsX86FmaSupported) yield return "fma"; - if (HardwareIntrinsics.IsX86LzcntSupported) yield return "lzcnt"; - if (HardwareIntrinsics.IsX86PclmulqdqSupported) yield return "pclmul"; - if (HardwareIntrinsics.IsX86PopcntSupported) yield return "popcnt"; + if (HardwareIntrinsics.IsX86Avx2Supported) + { + yield return "avx2"; + + if (runtimeMoniker <= RuntimeMoniker.NativeAot90) + { + yield return "bmi"; + yield return "bmi2"; + yield return "fma"; + yield return "lzcnt"; + } + } + if (HardwareIntrinsics.IsX86Avx512Supported && (runtimeMoniker > RuntimeMoniker.NativeAot80)) + { + if (runtimeMoniker >= RuntimeMoniker.NativeAot10_0) + { + yield return "avx512"; + } + else + { + yield return "avx512f"; + yield return "avx512f_vl"; + yield return "avx512bw"; + yield return "avx512bw_vl"; + yield return "avx512cd"; + yield return "avx512cd_vl"; + yield return "avx512dq"; + yield return "avx512dq_vl"; + } + } + if (HardwareIntrinsics.IsX86Avx512v2Supported && (runtimeMoniker > RuntimeMoniker.NativeAot80)) + { + if (runtimeMoniker >= RuntimeMoniker.NativeAot10_0) + { + yield return "avx512v2"; + } + else + { + yield return "avx512vbmi"; + yield return "avx512vbmi_vl"; + } + } + if (HardwareIntrinsics.IsX86Avx512v3Supported && (runtimeMoniker >= RuntimeMoniker.NativeAot10_0)) yield return "avx512v3"; + if (HardwareIntrinsics.IsX86Avx10v1Supported && (runtimeMoniker >= RuntimeMoniker.NativeAot90)) yield return "avx10v1"; + if (HardwareIntrinsics.IsX86Avx10v2Supported && (runtimeMoniker >= RuntimeMoniker.NativeAot10_0)) yield return "avx10v2"; + if (HardwareIntrinsics.IsX86AesSupported) + { + yield return "aes"; + if (runtimeMoniker <= RuntimeMoniker.NativeAot90) yield return "pclmul"; + } if (HardwareIntrinsics.IsX86AvxVnniSupported) yield return "avxvnni"; - if (HardwareIntrinsics.IsX86SerializeSupported) yield return "serialize"; + if (HardwareIntrinsics.IsX86SerializeSupported && runtimeMoniker > RuntimeMoniker.NativeAot70) yield return "serialize"; // https://github.com/dotnet/BenchmarkDotNet/issues/2463#issuecomment-1809625008 break; case Platform.Arm64: - if (HardwareIntrinsics.IsArmBaseSupported) yield return "base"; - if (HardwareIntrinsics.IsArmAdvSimdSupported) yield return "neon"; + if (HardwareIntrinsics.IsArmBaseSupported) + { + yield return "base"; + yield return "neon"; + } if (HardwareIntrinsics.IsArmAesSupported) yield return "aes"; if (HardwareIntrinsics.IsArmCrc32Supported) yield return "crc"; if (HardwareIntrinsics.IsArmDpSupported) yield return "dotprod"; if (HardwareIntrinsics.IsArmRdmSupported) yield return "rdma"; if (HardwareIntrinsics.IsArmSha1Supported) yield return "sha1"; if (HardwareIntrinsics.IsArmSha256Supported) yield return "sha2"; - // todo: handle "lse" break; default: yield break; diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs index 4703c4b657..9d476bf0e0 100644 --- a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs @@ -1,5 +1,8 @@ using System.Collections.Generic; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Validators; namespace BenchmarkDotNet.Toolchains.NativeAot { @@ -22,10 +25,10 @@ public class NativeAotToolchain : Toolchain .ToToolchain(); /// - /// compiled as net8.0, targets latest NativeAOT build from the .NET 8 feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json + /// compiled as net8.0, targets latest NativeAOT build from the NuGet.org feed: "https://api.nuget.org/v3/index.json" /// public static readonly IToolchain Net80 = CreateBuilder() - .UseNuGet("", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json") + .UseNuGet("", "https://api.nuget.org/v3/index.json") .TargetFrameworkMoniker("net8.0") .ToToolchain(); @@ -37,6 +40,14 @@ public class NativeAotToolchain : Toolchain .TargetFrameworkMoniker("net9.0") .ToToolchain(); + /// + /// compiled as net10.0, targets latest NativeAOT build from the .NET 10 feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json + /// + public static readonly IToolchain Net10_0 = CreateBuilder() + .UseNuGet("", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json") + .TargetFrameworkMoniker("net10.0") + .ToToolchain(); + internal NativeAotToolchain(string displayName, string ilCompilerVersion, string runtimeFrameworkVersion, string targetFrameworkMoniker, string runtimeIdentifier, @@ -60,5 +71,18 @@ internal NativeAotToolchain(string displayName, public static NativeAotToolchainBuilder CreateBuilder() => NativeAotToolchainBuilder.Create(); public static string GetExtraArguments(string runtimeIdentifier) => $"-r {runtimeIdentifier}"; + + public override IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) + { + foreach (var error in base.Validate(benchmarkCase, resolver)) + { + yield return error; + } + + foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(CustomDotNetCliPath, benchmarkCase)) + { + yield return validationError; + } + } } } diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchainBuilder.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchainBuilder.cs index 9cdc679716..1bd248e5eb 100644 --- a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchainBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchainBuilder.cs @@ -53,8 +53,8 @@ public NativeAotToolchainBuilder UseLocalBuild(DirectoryInfo ilcPackages) if (!ilcPackages.Exists) throw new DirectoryNotFoundException($"{ilcPackages} provided as {nameof(ilcPackages)} does NOT exist"); Feeds["local"] = ilcPackages.FullName; - ilCompilerVersion = "7.0.0-dev"; - Feeds["dotnet7"] = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json"; + ilCompilerVersion = "10.0.0-dev"; + Feeds["dotnet10"] = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json"; useTempFolderForRestore = true; DisplayName("local ILCompiler build"); diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs index 0718d6b8cb..3ba7054f7a 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; using System.Reflection; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; @@ -27,9 +28,9 @@ protected override string[] GetArtifactsToCleanup(ArtifactsPaths artifactsPaths) protected override void GenerateBuildScript(BuildPartition buildPartition, ArtifactsPaths artifactsPaths) { - string prefix = RuntimeInformation.IsWindows() ? "" : "#!/bin/bash\n"; + string prefix = OsDetector.IsWindows() ? "" : "#!/bin/bash\n"; var list = new List(); - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) list.Add("mono"); list.Add("csc"); list.Add("/noconfig"); diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs index 31c4d68c29..6774268dcc 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs @@ -51,12 +51,14 @@ public override IEnumerable Validate(BenchmarkCase benchmarkCas benchmarkCase); } +#pragma warning disable CS0618 // Type or member is obsolete if (benchmarkCase.Job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic)) { yield return new ValidationError(true, "The Roslyn toolchain does not allow specifying NuGet package dependencies", benchmarkCase); } +#pragma warning restore CS0618 // Type or member is obsolete } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/Toolchain.cs b/src/BenchmarkDotNet/Toolchains/Toolchain.cs index e680424de7..4ff773a797 100644 --- a/src/BenchmarkDotNet/Toolchains/Toolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Toolchain.cs @@ -57,31 +57,6 @@ public virtual IEnumerable Validate(BenchmarkCase benchmarkCase } } - internal static bool IsCliPathInvalid(string customDotNetCliPath, BenchmarkCase benchmarkCase, out ValidationError? validationError) - { - validationError = null; - - if (string.IsNullOrEmpty(customDotNetCliPath) && !HostEnvironmentInfo.GetCurrent().IsDotNetCliInstalled()) - { - validationError = new ValidationError(true, - $"BenchmarkDotNet requires dotnet SDK to be installed or path to local dotnet cli provided in explicit way using `--cli` argument, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", - benchmarkCase); - - return true; - } - - if (!string.IsNullOrEmpty(customDotNetCliPath) && !File.Exists(customDotNetCliPath)) - { - validationError = new ValidationError(true, - $"Provided custom dotnet cli path does not exist, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", - benchmarkCase); - - return true; - } - - return false; - } - public override string ToString() => Name; } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs index 837c757cd7..2beab5018c 100644 --- a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs +++ b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs @@ -1,4 +1,5 @@ using System; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; @@ -28,9 +29,7 @@ private static IToolchain GetToolchain(Job job, Descriptor descriptor) : GetToolchain( job.ResolveValue(EnvironmentMode.RuntimeCharacteristic, EnvironmentResolver.Instance), descriptor, - job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic) - || job.HasValue(InfrastructureMode.BuildConfigurationCharacteristic) - || job.HasValue(InfrastructureMode.ArgumentsCharacteristic)); + job.HasDynamicBuildCharacteristic()); internal static IToolchain GetToolchain(this Runtime runtime, Descriptor? descriptor = null, bool preferMsBuildToolchains = false) { @@ -48,9 +47,9 @@ internal static IToolchain GetToolchain(this Runtime runtime, Descriptor? descri : CsProjClassicNetToolchain.From(clrRuntime.MsBuildMoniker); case MonoRuntime mono: - if (RuntimeInformation.IsAndroid()) + if (OsDetector.IsAndroid()) return InProcessEmitToolchain.Instance; - if (RuntimeInformation.IsIOS()) + if (OsDetector.IsIOS()) return InProcessNoEmitToolchain.Instance; if (!string.IsNullOrEmpty(mono.AotArgs)) return MonoAotToolchain.Instance; @@ -67,6 +66,7 @@ internal static IToolchain GetToolchain(this Runtime runtime, Descriptor? descri RuntimeMoniker.Mono70 => GetToolchain(RuntimeMoniker.Net70), RuntimeMoniker.Mono80 => GetToolchain(RuntimeMoniker.Net80), RuntimeMoniker.Mono90 => GetToolchain(RuntimeMoniker.Net90), + RuntimeMoniker.Mono10_0 => GetToolchain(RuntimeMoniker.Net10_0), _ => CsProjCoreToolchain.From(new NetCoreAppSettings(mono.MsBuildMoniker, null, mono.Name)) }; } @@ -156,6 +156,9 @@ private static IToolchain GetToolchain(RuntimeMoniker runtimeMoniker) case RuntimeMoniker.Net90: return CsProjCoreToolchain.NetCoreApp90; + case RuntimeMoniker.Net10_0: + return CsProjCoreToolchain.NetCoreApp10_0; + case RuntimeMoniker.NativeAot60: return NativeAotToolchain.Net60; @@ -168,6 +171,9 @@ private static IToolchain GetToolchain(RuntimeMoniker runtimeMoniker) case RuntimeMoniker.NativeAot90: return NativeAotToolchain.Net90; + case RuntimeMoniker.NativeAot10_0: + return NativeAotToolchain.Net10_0; + case RuntimeMoniker.Mono60: return MonoToolchain.Mono60; @@ -180,6 +186,9 @@ private static IToolchain GetToolchain(RuntimeMoniker runtimeMoniker) case RuntimeMoniker.Mono90: return MonoToolchain.Mono90; + case RuntimeMoniker.Mono10_0: + return MonoToolchain.Mono10_0; + default: throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, "RuntimeMoniker not supported"); } diff --git a/src/BenchmarkDotNet/Validators/CompilationValidator.cs b/src/BenchmarkDotNet/Validators/CompilationValidator.cs index d8c2ec6c5b..b0bb87681d 100644 --- a/src/BenchmarkDotNet/Validators/CompilationValidator.cs +++ b/src/BenchmarkDotNet/Validators/CompilationValidator.cs @@ -7,6 +7,7 @@ using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains; using Microsoft.CodeAnalysis.CSharp; +using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Validators { @@ -25,8 +26,38 @@ private CompilationValidator() { } public IEnumerable Validate(ValidationParameters validationParameters) => ValidateCSharpNaming(validationParameters.Benchmarks) .Union(ValidateNamingConflicts(validationParameters.Benchmarks)) + .Union(ValidateClassModifiers((validationParameters.Benchmarks)) .Union(ValidateAccessModifiers(validationParameters.Benchmarks)) - .Union(ValidateBindingModifiers(validationParameters.Benchmarks)); + .Union(ValidateBindingModifiers(validationParameters.Benchmarks)) + ); + + private static IEnumerable ValidateClassModifiers(IEnumerable benchmarks) + { + return benchmarks + .Distinct(BenchmarkMethodEqualityComparer.Instance) + .SelectMany(benchmark => + { + var type = benchmark.Descriptor.Type; + var errors = new List(); + + if (type.IsSealed) + { + errors.Add(new ValidationError( + true, + $"Benchmarked method `{benchmark.Descriptor.WorkloadMethod.Name}` is within a sealed class, Declaring type must be unsealed.", + benchmark)); + } + if (!type.IsVisible) + { + errors.Add(new ValidationError( + true, + $"Benchmarked method `{benchmark.Descriptor.WorkloadMethod.Name}` is within a non-visible class, all declaring types must be public.", + benchmark)); + } + // TODO: Generics validation + return errors; + }); + } private static IEnumerable ValidateCSharpNaming(IEnumerable benchmarks) => benchmarks diff --git a/src/BenchmarkDotNet/Validators/ConfigCompatibilityValidator.cs b/src/BenchmarkDotNet/Validators/ConfigCompatibilityValidator.cs deleted file mode 100644 index 9732f40383..0000000000 --- a/src/BenchmarkDotNet/Validators/ConfigCompatibilityValidator.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System.Linq; -using System.Collections.Generic; -using BenchmarkDotNet.Reports; - -namespace BenchmarkDotNet.Validators -{ - public class ConfigCompatibilityValidator : IValidator - { - public static readonly ConfigCompatibilityValidator FailOnError = new ConfigCompatibilityValidator(); - - public bool TreatsWarningsAsErrors => true; - - public IEnumerable Validate(ValidationParameters validationParameters) - { - var orderers = - validationParameters - .Benchmarks - .Where(benchmark => benchmark.Config.Orderer != Order.DefaultOrderer.Instance) - .Select(benchmark => benchmark.Config.Orderer) - .Distinct(); - - if (orderers.Count() > 1) - yield return new ValidationError(true, "You use JoinSummary options, but provided configurations cannot be joined. Only one Orderer per benchmark cases is allowed."); - - var styles = - validationParameters - .Benchmarks - .Where(benchmark => benchmark.Config.SummaryStyle != SummaryStyle.Default - && benchmark.Config.SummaryStyle != null) // Paranoid - .Select(benchmark => benchmark.Config.SummaryStyle) - .Distinct(); - - if (styles.Count() > 1) - yield return new ValidationError(true, "You use JoinSummary options, but provided configurations cannot be joined. Only one SummaryStyle per benchmark cases is allowed."); - } - } -} diff --git a/src/BenchmarkDotNet/Validators/DotNetSdkValidator.cs b/src/BenchmarkDotNet/Validators/DotNetSdkValidator.cs new file mode 100644 index 0000000000..66b6a35aca --- /dev/null +++ b/src/BenchmarkDotNet/Validators/DotNetSdkValidator.cs @@ -0,0 +1,185 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; + +namespace BenchmarkDotNet.Validators +{ + internal static class DotNetSdkValidator + { + private static readonly Lazy> cachedFrameworkSdks = new Lazy>(GetInstalledFrameworkSdks, true); + + public static IEnumerable ValidateCoreSdks(string? customDotNetCliPath, BenchmarkCase benchmark) + { + if (IsCliPathInvalid(customDotNetCliPath, benchmark, out ValidationError? cliPathError)) + { + yield return cliPathError; + yield break; + } + var requiredSdkVersion = benchmark.GetRuntime().RuntimeMoniker.GetRuntimeVersion(); + if (!GetInstalledDotNetSdks(customDotNetCliPath).Any(sdk => sdk >= requiredSdkVersion)) + { + yield return new ValidationError(true, $"The required .NET Core SDK version {requiredSdkVersion} or higher for runtime moniker {benchmark.Job.Environment.Runtime.RuntimeMoniker} is not installed.", benchmark); + } + } + + public static IEnumerable ValidateFrameworkSdks(BenchmarkCase benchmark) + { + var requiredSdkVersion = benchmark.GetRuntime().RuntimeMoniker.GetRuntimeVersion(); + var installedVersionString = cachedFrameworkSdks.Value.FirstOrDefault(); + if (installedVersionString == null || Version.TryParse(installedVersionString, out var installedVersion) && installedVersion < requiredSdkVersion) + { + yield return new ValidationError(true, $"The required .NET Framework SDK version {requiredSdkVersion} or higher is not installed.", benchmark); + } + } + + public static bool IsCliPathInvalid(string customDotNetCliPath, BenchmarkCase benchmarkCase, out ValidationError? validationError) + { + validationError = null; + + if (string.IsNullOrEmpty(customDotNetCliPath) && !HostEnvironmentInfo.GetCurrent().IsDotNetCliInstalled()) + { + validationError = new ValidationError(true, + $"BenchmarkDotNet requires dotnet SDK to be installed or path to local dotnet cli provided in explicit way using `--cli` argument, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); + + return true; + } + + if (!string.IsNullOrEmpty(customDotNetCliPath) && !File.Exists(customDotNetCliPath)) + { + validationError = new ValidationError(true, + $"Provided custom dotnet cli path does not exist, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); + + return true; + } + + return false; + } + + private static IEnumerable GetInstalledDotNetSdks(string? customDotNetCliPath) + { + string dotnetExecutable = string.IsNullOrEmpty(customDotNetCliPath) ? "dotnet" : customDotNetCliPath; + var startInfo = new ProcessStartInfo(dotnetExecutable, "--list-sdks") + { + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + try + { + using (var process = Process.Start(startInfo)) + { + if (process == null) + { + return Enumerable.Empty(); + } + + process.WaitForExit(); + + if (process.ExitCode == 0) + { + var output = process.StandardOutput.ReadToEnd(); + var lines = output.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); + + var versions = new List(lines.Count()); + foreach (var line in lines) + { + // Version.TryParse does not handle things like 3.0.0-WORD, so this will get just the 3.0.0 part + var parsableVersionPart = CoreRuntime.GetParsableVersionPart(line); + if (Version.TryParse(parsableVersionPart, out var version)) + { + versions.Add(version); + } + } + + return versions; + } + else + { + return Enumerable.Empty(); + } + } + } + catch (Win32Exception) // dotnet CLI is not installed or not found in the path. + { + return Enumerable.Empty(); + } + } + + public static List GetInstalledFrameworkSdks() + { + var versions = new List(); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + Get45PlusFromRegistry(versions); + } + + return versions; + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Interoperability", "CA1416:Validate platform compatibility", Justification = "This code is protected with a runtime OS platform check")] + private static void Get45PlusFromRegistry(List versions) + { + const string subkey = @"SOFTWARE\Microsoft\NET Framework Setup\NDP\v4\Full\"; + + using (var ndpKey = Microsoft.Win32.Registry.LocalMachine.OpenSubKey(subkey)) + { + if (ndpKey == null) + { + return; + } + + if (ndpKey.GetValue("Version") != null) + { + versions.Add(ndpKey.GetValue("Version").ToString()); + } + else + { + if (ndpKey.GetValue("Release") != null) + { + versions.Add(CheckFor45PlusVersion((int)ndpKey.GetValue("Release"))); + } + } + } + } + + private static string CheckFor45PlusVersion(int releaseKey) + { + if (releaseKey >= 533320) + return "4.8.1"; + if (releaseKey >= 528040) + return "4.8"; + if (releaseKey >= 461808) + return "4.7.2"; + if (releaseKey >= 461308) + return "4.7.1"; + if (releaseKey >= 460798) + return "4.7"; + if (releaseKey >= 394802) + return "4.6.2"; + if (releaseKey >= 394254) + return "4.6.1"; + if (releaseKey >= 393295) + return "4.6"; + if (releaseKey >= 379893) + return "4.5.2"; + if (releaseKey >= 378675) + return "4.5.1"; + if (releaseKey >= 378389) + return "4.5"; + + return ""; + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Validators/ExecutionValidatorBase.cs b/src/BenchmarkDotNet/Validators/ExecutionValidatorBase.cs index 11af691440..bb94aec254 100644 --- a/src/BenchmarkDotNet/Validators/ExecutionValidatorBase.cs +++ b/src/BenchmarkDotNet/Validators/ExecutionValidatorBase.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Validators @@ -130,21 +131,8 @@ private void TryToGetTaskResult(object result) return; } - var returnType = result.GetType(); - if (returnType.IsGenericType && returnType.GetGenericTypeDefinition() == typeof(ValueTask<>)) - { - var asTaskMethod = result.GetType().GetMethod("AsTask"); - result = asTaskMethod.Invoke(result, null); - } - - if (result is Task task) - { - task.GetAwaiter().GetResult(); - } - else if (result is ValueTask valueTask) - { - valueTask.GetAwaiter().GetResult(); - } + AwaitHelper.GetGetResultMethod(result.GetType()) + ?.Invoke(null, new[] { result }); } private bool TryToSetParamsFields(object benchmarkTypeInstance, List errors) diff --git a/src/BenchmarkDotNet/Validators/ParamsValidator.cs b/src/BenchmarkDotNet/Validators/ParamsValidator.cs index 630811b62b..37dc645af5 100644 --- a/src/BenchmarkDotNet/Validators/ParamsValidator.cs +++ b/src/BenchmarkDotNet/Validators/ParamsValidator.cs @@ -9,7 +9,7 @@ namespace BenchmarkDotNet.Validators { public class ParamsValidator : IValidator { - public static readonly ParamsValidator FailOnError = new (); + public static readonly ParamsValidator FailOnError = new(); public bool TreatsWarningsAsErrors => true; @@ -24,19 +24,19 @@ private IEnumerable Validate(Type type) BindingFlags.FlattenHierarchy; foreach (var memberInfo in type.GetMembers(reflectionFlags)) { - var attributes = new Attribute[] + var attributes = new Attribute?[] { memberInfo.ResolveAttribute(), memberInfo.ResolveAttribute(), memberInfo.ResolveAttribute() } - .Where(attribute => attribute != null) + .WhereNotNull() .ToList(); if (attributes.IsEmpty()) continue; string name = $"{type.Name}.{memberInfo.Name}"; - string? attributeString = string.Join(", ", attributes.Select(attribute => $"[{attribute.GetType().Name.Replace(nameof(Attribute), "")}]")); + string attributeString = string.Join(", ", attributes.Select(attribute => $"[{attribute.GetType().Name.Replace(nameof(Attribute), "")}]")); if (attributes.Count > 1) yield return new ValidationError(TreatsWarningsAsErrors, diff --git a/src/BenchmarkDotNet/Validators/RuntimeValidator.cs b/src/BenchmarkDotNet/Validators/RuntimeValidator.cs new file mode 100644 index 0000000000..debadf880e --- /dev/null +++ b/src/BenchmarkDotNet/Validators/RuntimeValidator.cs @@ -0,0 +1,44 @@ +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Toolchains; + +namespace BenchmarkDotNet.Validators; + +/// +/// Validator for runtime characteristic. +/// +/// +public class RuntimeValidator : IValidator +{ + public static readonly IValidator DontFailOnError = new RuntimeValidator(); + + private RuntimeValidator() { } + + public bool TreatsWarningsAsErrors => false; + + public IEnumerable Validate(ValidationParameters input) + { + var allBenchmarks = input.Benchmarks.ToArray(); + var nullRuntimeBenchmarks = allBenchmarks.Where(x => x.Job.Environment.Runtime == null).ToArray(); + + // There is no validation error if all the runtimes are set or if all the runtimes are null. + if (allBenchmarks.Length == nullRuntimeBenchmarks.Length) + { + return []; + } + + var errors = new List(); + foreach (var benchmark in nullRuntimeBenchmarks.Where(x=> !x.GetToolchain().IsInProcess)) + { + var job = benchmark.Job; + var jobText = job.HasValue(CharacteristicObject.IdCharacteristic) + ? job.Id + : CharacteristicSetPresenter.Display.ToPresentation(job); // Use job text representation instead for auto generated JobId. + + var message = $"Job({jobText}) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly."; + errors.Add(new ValidationError(false, message)); + } + return errors; + } +} diff --git a/src/BenchmarkDotNet/Validators/ValidationError.cs b/src/BenchmarkDotNet/Validators/ValidationError.cs index 23faff5a54..9bec081479 100644 --- a/src/BenchmarkDotNet/Validators/ValidationError.cs +++ b/src/BenchmarkDotNet/Validators/ValidationError.cs @@ -15,11 +15,11 @@ public ValidationError(bool isCritical, string message, BenchmarkCase? benchmark [PublicAPI] public bool IsCritical { get; } [PublicAPI] public string Message { get; } - [PublicAPI] public BenchmarkCase BenchmarkCase { get; } + [PublicAPI] public BenchmarkCase? BenchmarkCase { get; } public override string ToString() => Message; - public bool Equals(ValidationError other) + public bool Equals(ValidationError? other) { if (ReferenceEquals(null, other)) return false; @@ -28,13 +28,13 @@ public bool Equals(ValidationError other) return IsCritical == other.IsCritical && string.Equals(Message, other.Message) && Equals(BenchmarkCase, other.BenchmarkCase); } - public override bool Equals(object obj) + public override bool Equals(object? obj) { if (ReferenceEquals(null, obj)) return false; if (ReferenceEquals(this, obj)) return true; - if (obj.GetType() != this.GetType()) + if (obj.GetType() != GetType()) return false; return Equals((ValidationError)obj); } diff --git a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/.template.config/template.json b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/.template.config/template.json index d16cc38bf0..db2ededcbb 100644 --- a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/.template.config/template.json +++ b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/.template.config/template.json @@ -34,6 +34,14 @@ "description": "The target framework for the project.", "datatype": "choice", "choices": [ + { + "choice": "net10.0", + "description": ".NET 10" + }, + { + "choice": "net9.0", + "description": ".NET 9" + }, { "choice": "net8.0", "description": ".NET 8" @@ -91,7 +99,7 @@ "cases": [ { "condition": "(consoleApp == true)", - "value": "net6.0" + "value": "net8.0" }, { "condition": "(consoleApp == false)", @@ -131,7 +139,7 @@ "type": "parameter", "datatype": "string", "description": "Version of BenchmarkDotNet that will be referenced.", - "defaultValue": "0.13.10", + "defaultValue": "0.15.4", "replaces": "$(BenchmarkDotNetVersion)" } }, @@ -176,4 +184,4 @@ "continueOnError": true } ] -} \ No newline at end of file +} diff --git a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/.template.config/template.json b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/.template.config/template.json index 2a1e275699..e18fc9a0a6 100644 --- a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/.template.config/template.json +++ b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/.template.config/template.json @@ -34,6 +34,14 @@ "description": "The target framework for the project.", "datatype": "choice", "choices": [ + { + "choice": "net10.0", + "description": ".NET 10" + }, + { + "choice": "net9.0", + "description": ".NET 9" + }, { "choice": "net8.0", "description": ".NET 8" @@ -91,7 +99,7 @@ "cases": [ { "condition": "(consoleApp == true)", - "value": "net6.0" + "value": "net8.0" }, { "condition": "(consoleApp == false)", @@ -131,7 +139,7 @@ "type": "parameter", "datatype": "string", "description": "Version of BenchmarkDotNet that will be referenced.", - "defaultValue": "0.13.10", + "defaultValue": "0.15.4", "replaces": "$(BenchmarkDotNetVersion)" } }, @@ -176,4 +184,4 @@ "continueOnError": true } ] -} \ No newline at end of file +} diff --git a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.VB/.template.config/template.json b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.VB/.template.config/template.json index feb22ba4dd..67c362a8fb 100644 --- a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.VB/.template.config/template.json +++ b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.VB/.template.config/template.json @@ -34,6 +34,14 @@ "description": "The target framework for the project.", "datatype": "choice", "choices": [ + { + "choice": "net10.0", + "description": ".NET 10" + }, + { + "choice": "net9.0", + "description": ".NET 9" + }, { "choice": "net8.0", "description": ".NET 8" @@ -91,7 +99,7 @@ "cases": [ { "condition": "(consoleApp == true)", - "value": "net6.0" + "value": "net8.0" }, { "condition": "(consoleApp == false)", @@ -131,7 +139,7 @@ "type": "parameter", "datatype": "string", "description": "Version of BenchmarkDotNet that will be referenced.", - "defaultValue": "0.13.10", + "defaultValue": "0.15.4", "replaces": "$(BenchmarkDotNetVersion)" } }, @@ -176,4 +184,4 @@ "continueOnError": true } ] -} \ No newline at end of file +} diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj new file mode 100644 index 0000000000..6440cd9059 --- /dev/null +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj @@ -0,0 +1,26 @@ + + + + net8.0;net462 + false + true + true + false + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs new file mode 100644 index 0000000000..8dd4a4443c --- /dev/null +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs @@ -0,0 +1,297 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Builders; +using BenchmarkDotNet.Tests.Mocks; +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Exporters.Plotting.Tests +{ + public class ScottPlotExporterTests(ITestOutputHelper output) + { + public static TheoryData GetGroupBenchmarkTypes() + { + var data = new TheoryData(); + foreach (var type in typeof(BaselinesBenchmarks).GetNestedTypes()) + data.Add(type); + return data; + } + + [Theory] + [MemberData(nameof(GetGroupBenchmarkTypes))] + public void BarPlots(Type benchmarkType) + { + var logger = new AccumulationLogger(); + logger.WriteLine("=== " + benchmarkType.Name + " ==="); + + var exporter = new ScottPlotExporter() + { + IncludeBarPlot = true, + IncludeBoxPlot = false, + }; + var summary = MockFactory.CreateSummary(benchmarkType); + var filePaths = exporter.ExportToFiles(summary, logger).ToList(); + Assert.NotEmpty(filePaths); + Assert.All(filePaths, f => File.Exists(f)); + + foreach (string filePath in filePaths) + logger.WriteLine($"* {filePath}"); + output.WriteLine(logger.GetLog()); + } + + [Theory] + [MemberData(nameof(GetGroupBenchmarkTypes))] + public void BoxPlots(Type benchmarkType) + { + var logger = new AccumulationLogger(); + logger.WriteLine("=== " + benchmarkType.Name + " ==="); + + var exporter = new ScottPlotExporter() + { + IncludeBarPlot = false, + IncludeBoxPlot = true, + }; + var summary = MockFactory.CreateSummaryWithBiasedDistribution(benchmarkType, 1, 4, 10, 9); + var filePaths = exporter.ExportToFiles(summary, logger).ToList(); + Assert.NotEmpty(filePaths); + Assert.All(filePaths, f => File.Exists(f)); + + foreach (string filePath in filePaths) + logger.WriteLine($"* {filePath}"); + output.WriteLine(logger.GetLog()); + } + + [Theory] + [MemberData(nameof(GetGroupBenchmarkTypes))] + public void BoxPlotsWithOneMeasurement(Type benchmarkType) + { + var logger = new AccumulationLogger(); + logger.WriteLine("=== " + benchmarkType.Name + " ==="); + + var exporter = new ScottPlotExporter() + { + IncludeBarPlot = false, + IncludeBoxPlot = true, + }; + var summary = MockFactory.CreateSummaryWithBiasedDistribution(benchmarkType, 1, 4, 10, 1); + var filePaths = exporter.ExportToFiles(summary, logger).ToList(); + Assert.NotEmpty(filePaths); + Assert.All(filePaths, f => File.Exists(f)); + + foreach (string filePath in filePaths) + logger.WriteLine($"* {filePath}"); + output.WriteLine(logger.GetLog()); + } + + [SuppressMessage("ReSharper", "InconsistentNaming")] + public static class BaselinesBenchmarks + { + /* NoBaseline */ + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + public class NoBaseline_MethodsParamsJobs + { + [Params(2, 10)] public int Param; + + [Benchmark] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByMethod)] + public class NoBaseline_MethodsParamsJobs_GroupByMethod + { + [Params(2, 10)] public int Param; + + [Benchmark, BenchmarkCategory("CatA")] public void Base() { } + [Benchmark, BenchmarkCategory("CatB")] public void Foo() { } + [Benchmark, BenchmarkCategory("CatB")] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByJob)] + public class NoBaseline_MethodsParamsJobs_GroupByJob + { + [Params(2, 10)] public int Param; + + [Benchmark, BenchmarkCategory("CatA")] public void Base() { } + [Benchmark, BenchmarkCategory("CatB")] public void Foo() { } + [Benchmark, BenchmarkCategory("CatB")] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByParams)] + public class NoBaseline_MethodsParamsJobs_GroupByParams + { + [Params(2, 10)] public int Param; + + [Benchmark, BenchmarkCategory("CatA")] public void Base() { } + [Benchmark, BenchmarkCategory("CatB")] public void Foo() { } + [Benchmark, BenchmarkCategory("CatB")] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] + public class NoBaseline_MethodsParamsJobs_GroupByCategory + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true), BenchmarkCategory("CatA")] + public void A1() { } + + [Benchmark, BenchmarkCategory("CatA")] public void A2() { } + + [Benchmark(Baseline = true), BenchmarkCategory("CatB")] + public void B1() { } + + [Benchmark, BenchmarkCategory("CatB")] public void B2() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy( + BenchmarkLogicalGroupRule.ByMethod, + BenchmarkLogicalGroupRule.ByJob, + BenchmarkLogicalGroupRule.ByParams, + BenchmarkLogicalGroupRule.ByCategory)] + public class NoBaseline_MethodsParamsJobs_GroupByAll + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true), BenchmarkCategory("CatA")] + public void A1() { } + + [Benchmark, BenchmarkCategory("CatA")] public void A2() { } + + [Benchmark(Baseline = true), BenchmarkCategory("CatB")] + public void B1() { } + + [Benchmark, BenchmarkCategory("CatB")] public void B2() { } + } + + /* MethodBaseline */ + + [RankColumn, LogicalGroupColumn, BaselineColumn] + public class MethodBaseline_Methods + { + [Benchmark(Baseline = true)] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + public class MethodBaseline_MethodsParams + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true)] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + public class MethodBaseline_MethodsJobs + { + [Benchmark(Baseline = true)] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + public class MethodBaseline_MethodsParamsJobs + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true)] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + /* JobBaseline */ + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] + public class JobBaseline_MethodsJobs + { + [Benchmark] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] + public class JobBaseline_MethodsParamsJobs + { + [Params(2, 10)] public int Param; + + [Benchmark] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + /* MethodJobBaseline */ + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] + public class MethodJobBaseline_MethodsJobs + { + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] + public class MethodJobBaseline_MethodsJobsParams + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark] public void Bar() { } + } + + /* Invalid */ + + [RankColumn, LogicalGroupColumn, BaselineColumn] + public class Invalid_TwoMethodBaselines + { + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark(Baseline = true)] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2", baseline: true)] + public class Invalid_TwoJobBaselines + { + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + /* Escape Params */ + + public class Escape_ParamsAndArguments + { + [Params("\t", "\n")] public string StringParam; + + [Arguments('\t')] + [Arguments('\n')] + [Benchmark] public void Foo(char charArg) { } + + [Benchmark] public void Bar() { } + } + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj b/tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj index 70ca871259..e78f71c447 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj @@ -2,7 +2,7 @@ BenchmarkDotNet.IntegrationTests.ConfigPerAssembly - net462;net7.0 + net462;net8.0 true BenchmarkDotNet.IntegrationTests.ConfigPerAssembly BenchmarkDotNet.IntegrationTests.ConfigPerAssembly @@ -14,7 +14,7 @@ - + \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.CustomPaths/BenchmarkDotNet.IntegrationTests.CustomPaths.csproj b/tests/BenchmarkDotNet.IntegrationTests.CustomPaths/BenchmarkDotNet.IntegrationTests.CustomPaths.csproj index cb4df7361e..326690a5b6 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.CustomPaths/BenchmarkDotNet.IntegrationTests.CustomPaths.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.CustomPaths/BenchmarkDotNet.IntegrationTests.CustomPaths.csproj @@ -14,7 +14,7 @@ - + diff --git a/tests/BenchmarkDotNet.IntegrationTests.DisabledOptimizations/BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj b/tests/BenchmarkDotNet.IntegrationTests.DisabledOptimizations/BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj index aad9d6ac33..6ceec4010a 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.DisabledOptimizations/BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.DisabledOptimizations/BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj @@ -2,7 +2,7 @@ BenchmarkDotNet.IntegrationTests.DisabledOptimizations - net462;net7.0 + net462;net8.0 true BenchmarkDotNet.IntegrationTests.DisabledOptimizations BenchmarkDotNet.IntegrationTests.DisabledOptimizations @@ -16,7 +16,7 @@ - + \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.EnabledOptimizations/BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj b/tests/BenchmarkDotNet.IntegrationTests.EnabledOptimizations/BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj index da7558c82e..28bf3ef4f9 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.EnabledOptimizations/BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.EnabledOptimizations/BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj @@ -2,7 +2,7 @@ BenchmarkDotNet.IntegrationTests.EnabledOptimizations - net462;net7.0 + net462;net8.0 true BenchmarkDotNet.IntegrationTests.EnabledOptimizations BenchmarkDotNet.IntegrationTests.EnabledOptimizations @@ -16,7 +16,7 @@ - + \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.FSharp/BenchmarkDotNet.IntegrationTests.FSharp.fsproj b/tests/BenchmarkDotNet.IntegrationTests.FSharp/BenchmarkDotNet.IntegrationTests.FSharp.fsproj index 6feefadc37..505a549daf 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.FSharp/BenchmarkDotNet.IntegrationTests.FSharp.fsproj +++ b/tests/BenchmarkDotNet.IntegrationTests.FSharp/BenchmarkDotNet.IntegrationTests.FSharp.fsproj @@ -5,7 +5,7 @@ - net462;net7.0 + net462;net8.0 false @@ -15,7 +15,7 @@ - - + + \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj index e7fadcfc16..37dfe950f7 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj @@ -3,8 +3,12 @@ BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks + net461;net48;netcoreapp2.0;net8.0 + + $(NoWarn);NETSDK1138;NU1903 + + $(NoWarn);NU1510 false - net461;net48;netcoreapp2.0;net7.0 true BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks @@ -13,6 +17,8 @@ AnyCPU true true + + true @@ -23,8 +29,7 @@ - - + @@ -37,9 +42,10 @@ - - - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs index 701b7d7ee1..b9cf360c8f 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs @@ -5,7 +5,7 @@ using BenchmarkDotNet.Jobs; using Xunit; -namespace BenchmarkDotNet.IntegrationTests +namespace BenchmarkDotNet.IntegrationTests.ManualRunning { public class MultipleFrameworksTest : BenchmarkTestExecutor { @@ -15,9 +15,23 @@ public class MultipleFrameworksTest : BenchmarkTestExecutor [InlineData(RuntimeMoniker.Net461)] [InlineData(RuntimeMoniker.Net48)] [InlineData(RuntimeMoniker.NetCoreApp20)] - [InlineData(RuntimeMoniker.Net70)] + [InlineData(RuntimeMoniker.Net80)] public void EachFrameworkIsRebuilt(RuntimeMoniker runtime) { +#if NET461 + // We cannot detect what target framework version the host was compiled for on full Framework, + // which causes the RoslynToolchain to be used instead of CsProjClassicNetToolchain when the host is full Framework + // (because full Framework always uses the version that's installed on the machine, unlike Core), + // which means if the machine has net48 installed (not net481), the net461 host with net48 runtime moniker + // will not be recompiled, causing the test to fail. + + // If we ever change the default toolchain to CsProjClassicNetToolchain instead of RoslynToolchain, we can remove this check. + if (runtime == RuntimeMoniker.Net48) + { + // XUnit doesn't provide Assert.Skip API yet. + return; + } +#endif var config = ManualConfig.CreateEmpty().AddJob(Job.Dry.WithRuntime(runtime.GetRuntime()).WithEnvironmentVariable(TfmEnvVarName, runtime.ToString())); CanExecute(config); } @@ -31,8 +45,8 @@ public class ValuePerTfm RuntimeMoniker.Net48; #elif NETCOREAPP2_0 RuntimeMoniker.NetCoreApp20; -#elif NET7_0 - RuntimeMoniker.Net70; +#elif NET8_0 + RuntimeMoniker.Net80; #else RuntimeMoniker.NotRecognized; #endif @@ -42,7 +56,7 @@ public void ThrowWhenWrong() { if (Environment.GetEnvironmentVariable(TfmEnvVarName) != moniker.ToString()) { - throw new InvalidOperationException($"Has not been recompiled, the value was {Environment.GetEnvironmentVariable(TfmEnvVarName)}"); + throw new InvalidOperationException($"Has not been recompiled, the value was {moniker}, expected {Environment.GetEnvironmentVariable(TfmEnvVarName)}"); } } } diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj index 136d505b6f..3e1859a72c 100755 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj @@ -1,8 +1,8 @@ - + BenchmarkDotNet.IntegrationTests.ManualRunning - net462;net7.0 + net462;net8.0 true BenchmarkDotNet.IntegrationTests.ManualRunning BenchmarkDotNet.IntegrationTests.ManualRunning @@ -11,6 +11,11 @@ AnyCPU true + + + $(DefineConstants);CUSTOM_PROP + + @@ -29,9 +34,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs index 4fd4108973..29bf8275dd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Extensions; @@ -13,7 +14,7 @@ using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.InProcess.Emit; using Perfolizer.Horology; -using Perfolizer.Mathematics.Thresholds; +using Perfolizer.Metrology; using Xunit; using Xunit.Abstractions; @@ -110,20 +111,17 @@ private void AssertZeroResults(Type benchmarkType, IConfig config) .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) ); - var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue; - var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds); + var cpuResolution = CpuDetector.Cpu?.MaxFrequency()?.ToResolution() ?? FallbackCpuResolutionValue; + var threshold = new NumberValue(cpuResolution.Nanoseconds).ToThreshold(); foreach (var report in summary.Reports) { - var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers(); - var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers(); + var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().Sample; + var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().Sample; - bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold); + bool isZero = ZeroMeasurementHelper.AreIndistinguishable(workloadMeasurements, overheadMeasurements, threshold); Assert.True(isZero, $"Actual time was not 0."); - isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold); - Assert.True(isZero, "Overhead took more time than workload."); - Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0."); } } @@ -166,20 +164,17 @@ private void AssertDifferentSizedStructsResults(IConfig config) .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) ); - var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue; - var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds); + var cpuResolution = CpuDetector.Cpu?.MaxFrequency()?.ToResolution() ?? FallbackCpuResolutionValue; + var threshold = (cpuResolution / 2).ToThreshold(); foreach (var report in summary.Reports) { - var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers(); - var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers(); + var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().Sample; + var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().Sample; - bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold); + bool isZero = ZeroMeasurementHelper.AreIndistinguishable(workloadMeasurements, overheadMeasurements, threshold); Assert.False(isZero, $"Actual time was 0."); - isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold); - Assert.True(isZero, "Overhead took more time than workload."); - Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0."); } } @@ -197,16 +192,12 @@ public struct Struct32 public struct Struct64 { - public long l1, l2, l3, l4, - l5, l6, l7, l8; + public long l1, l2, l3, l4, l5, l6, l7, l8; } public struct Struct128 { - public long l1, l2, l3, l4, - l5, l6, l7, l8, - l9, l10, l11, l12, - l13, l14, l15, l16; + public long l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15, l16; } public class DifferentSizedStructs @@ -218,17 +209,72 @@ public class DifferentSizedStructs } } -public class EmptyVoid { [Benchmark] public void Void() { } } -public class EmptyByte { [Benchmark] public byte Byte() => default; } -public class EmptySByte { [Benchmark] public sbyte SByte() => default; } -public class EmptyShort { [Benchmark] public short Short() => default; } -public class EmptyUShort { [Benchmark] public ushort UShort() => default; } -public class EmptyChar { [Benchmark] public char Char() => default; } -public class EmptyInt32 { [Benchmark] public int Int32() => default; } -public class EmptyUInt32 { [Benchmark] public uint UInt32() => default; } -public class EmptyInt64 { [Benchmark] public long Int64() => default; } -public class EmptyUInt64 { [Benchmark] public ulong UInt64() => default; } -public class EmptyIntPtr { [Benchmark] public IntPtr IntPtr() => default; } -public class EmptyUIntPtr { [Benchmark] public UIntPtr UIntPtr() => default; } -public class EmptyVoidPointer { [Benchmark] public unsafe void* VoidPointer() => default; } -public class EmptyClass { [Benchmark] public object Class() => default; } \ No newline at end of file +public class EmptyVoid +{ + [Benchmark] public void Void() { } +} + +public class EmptyByte +{ + [Benchmark] public byte Byte() => default; +} + +public class EmptySByte +{ + [Benchmark] public sbyte SByte() => default; +} + +public class EmptyShort +{ + [Benchmark] public short Short() => default; +} + +public class EmptyUShort +{ + [Benchmark] public ushort UShort() => default; +} + +public class EmptyChar +{ + [Benchmark] public char Char() => default; +} + +public class EmptyInt32 +{ + [Benchmark] public int Int32() => default; +} + +public class EmptyUInt32 +{ + [Benchmark] public uint UInt32() => default; +} + +public class EmptyInt64 +{ + [Benchmark] public long Int64() => default; +} + +public class EmptyUInt64 +{ + [Benchmark] public ulong UInt64() => default; +} + +public class EmptyIntPtr +{ + [Benchmark] public IntPtr IntPtr() => default; +} + +public class EmptyUIntPtr +{ + [Benchmark] public UIntPtr UIntPtr() => default; +} + +public class EmptyVoidPointer +{ + [Benchmark] public unsafe void* VoidPointer() => default; +} + +public class EmptyClass +{ + [Benchmark] public object Class() => default; +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalNativeAotToolchainTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalNativeAotToolchainTests.cs index c8740a3829..64f469efc7 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalNativeAotToolchainTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalNativeAotToolchainTests.cs @@ -13,7 +13,7 @@ namespace BenchmarkDotNet.IntegrationTests.ManualRunning /// to run these tests please clone and build NativeAOT first, /// then update the hardcoded path /// and run following command from console: - /// dotnet test -c Release -f netcoreapp2.1 --filter "FullyQualifiedName~BenchmarkDotNet.IntegrationTests.ManualRunning.LocalNativeAotToolchainTests" + /// dotnet test -c Release -f net8.0 --filter "FullyQualifiedName~BenchmarkDotNet.IntegrationTests.ManualRunning.LocalNativeAotToolchainTests" /// /// in perfect world we would do this OOB for you, but building NativeAOT /// so it's not part of our CI jobs @@ -29,7 +29,7 @@ public void CanBenchmarkLocalBuildUsingRyuJit() { var config = ManualConfig.CreateEmpty() .AddJob(Job.Dry - .WithRuntime(NativeAotRuntime.Net60) + .WithRuntime(NativeAotRuntime.Net80) .WithToolchain( NativeAotToolchain.CreateBuilder() .UseLocalBuild(new System.IO.DirectoryInfo(IlcPath)) diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/MsBuildArgumentTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/MsBuildArgumentTests.cs new file mode 100644 index 0000000000..f4619040cc --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/MsBuildArgumentTests.cs @@ -0,0 +1,65 @@ +using System; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using Xunit; + +namespace BenchmarkDotNet.IntegrationTests.ManualRunning +{ + public class MsBuildArgumentTests : BenchmarkTestExecutor + { + private const string CustomPropEnvVarName = "CustomPropEnvVarName"; + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ProcessIsBuiltWithCustomProperty(bool setCustomProperty) + { + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry + .WithArguments(new Argument[] { new MsBuildArgument($"/p:CustomProp={setCustomProperty}") }) + .WithEnvironmentVariable(CustomPropEnvVarName, setCustomProperty.ToString()) + ); + CanExecute(config); + } + + [Fact] + public void MultipleProcessesAreBuiltWithCorrectProperties() + { + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry + .WithArguments(new Argument[] { new MsBuildArgument($"/p:CustomProp={true}") }) + .WithEnvironmentVariable(CustomPropEnvVarName, true.ToString()) + ) + .AddJob(Job.Dry + .WithRuntime(NativeAotRuntime.Net80) + .WithArguments(new Argument[] { new MsBuildArgument($"/p:CustomProp={true}") }) + .WithEnvironmentVariable(CustomPropEnvVarName, true.ToString()) + ) + .AddJob(Job.Dry + .WithEnvironmentVariable(CustomPropEnvVarName, false.ToString()) + ); + CanExecute(config); + } + + public class PropertyDefine + { + private const bool customPropWasSet = +#if CUSTOM_PROP + true; +#else + false; +#endif + + [Benchmark] + public void ThrowWhenWrong() + { + if (Environment.GetEnvironmentVariable(CustomPropEnvVarName) != customPropWasSet.ToString()) + { + throw new InvalidOperationException($"Custom property was not set properly, the expected value was {Environment.GetEnvironmentVariable(CustomPropEnvVarName)}"); + } + } + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeEmpty.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeEmpty.cs deleted file mode 100644 index f8607e4263..0000000000 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeEmpty.cs +++ /dev/null @@ -1,85 +0,0 @@ -using BenchmarkDotNet.Attributes; - -namespace BenchmarkDotNet.IntegrationTests.ManualRunning.Smoke -{ - [MedianColumn, Q3Column, MaxColumn] - [LegacyJitX64Job, RyuJitX64Job, MonoJob] - [KeepBenchmarkFiles] - public class SmokeEmpty - { - [Benchmark] public void Void1() {} - [Benchmark] public void Void2() {} - [Benchmark] public void Void3() {} - [Benchmark] public void Void4() {} - - [Benchmark] public byte Byte1() => 0; - [Benchmark] public byte Byte2() => 0; - [Benchmark] public byte Byte3() => 0; - [Benchmark] public byte Byte4() => 0; - - [Benchmark] public sbyte Sbyte1() => 0; - [Benchmark] public sbyte Sbyte2() => 0; - [Benchmark] public sbyte Sbyte3() => 0; - [Benchmark] public sbyte Sbyte4() => 0; - - [Benchmark] public short Short1() => 0; - [Benchmark] public short Short2() => 0; - [Benchmark] public short Short3() => 0; - [Benchmark] public short Short4() => 0; - - [Benchmark] public ushort Ushort1() => 0; - [Benchmark] public ushort Ushort2() => 0; - [Benchmark] public ushort Ushort3() => 0; - [Benchmark] public ushort Ushort4() => 0; - - [Benchmark] public int Int1() => 0; - [Benchmark] public int Int2() => 0; - [Benchmark] public int Int3() => 0; - [Benchmark] public int Int4() => 0; - - [Benchmark] public uint Uint1() => 0u; - [Benchmark] public uint Uint2() => 0u; - [Benchmark] public uint Uint3() => 0u; - [Benchmark] public uint Uint4() => 0u; - - [Benchmark] public bool Bool1() => false; - [Benchmark] public bool Bool2() => false; - [Benchmark] public bool Bool3() => false; - [Benchmark] public bool Bool4() => false; - - [Benchmark] public char Char1() => 'a'; - [Benchmark] public char Char2() => 'a'; - [Benchmark] public char Char3() => 'a'; - [Benchmark] public char Char4() => 'a'; - - [Benchmark] public float Float1() => 0f; - [Benchmark] public float Float2() => 0f; - [Benchmark] public float Float3() => 0f; - [Benchmark] public float Float4() => 0f; - - [Benchmark] public double Double1() => 0d; - [Benchmark] public double Double2() => 0d; - [Benchmark] public double Double3() => 0d; - [Benchmark] public double Double4() => 0d; - - [Benchmark] public long Long1() => 0L; - [Benchmark] public long Long2() => 0L; - [Benchmark] public long Long3() => 0L; - [Benchmark] public long Long4() => 0L; - - [Benchmark] public ulong Ulong1() => 0uL; - [Benchmark] public ulong Ulong2() => 0uL; - [Benchmark] public ulong Ulong3() => 0uL; - [Benchmark] public ulong Ulong4() => 0uL; - - [Benchmark] public string String1() => ""; - [Benchmark] public string String2() => ""; - [Benchmark] public string String3() => ""; - [Benchmark] public string String4() => ""; - - [Benchmark] public object Object1() => null; - [Benchmark] public object Object2() => null; - [Benchmark] public object Object3() => null; - [Benchmark] public object Object4() => null; - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeValueTypes.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeValueTypes.cs deleted file mode 100644 index 34b93a6d75..0000000000 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeValueTypes.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System; -using System.Threading.Tasks; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Environments; - -namespace BenchmarkDotNet.IntegrationTests.ManualRunning.Smoke -{ - // ReSharper disable InconsistentNaming - - [RyuJitX64Job, LegacyJitX64Job, LegacyJitX86Job] - [MemoryDiagnoser] - public class SmokeValueTypes - { - [Benchmark] public Jit ReturnEnum() => Jit.RyuJit; - - [Benchmark] public DateTime ReturnDateTime() => new DateTime(); - - [Benchmark] public DateTime? ReturnNullableDateTime() => new DateTime(); - [Benchmark] public int? ReturnNullableInt() => 0; - - public struct StructWithReferencesOnly { public object _ref; } - [Benchmark] public StructWithReferencesOnly ReturnStructWithReferencesOnly() => new StructWithReferencesOnly(); - - public struct EmptyStruct { } - [Benchmark] public EmptyStruct ReturnEmptyStruct() => new EmptyStruct(); - - [Benchmark] public ValueTuple ReturnGenericStructOfValueType() => new ValueTuple(0); - [Benchmark] public ValueTuple ReturnGenericStructOfReferenceType() => new ValueTuple(null); - - [Benchmark] public ValueTask ReturnValueTaskOfValueType() => new ValueTask(0); - [Benchmark] public ValueTask ReturnValueTaskOfReferenceType() => new ValueTask(result: null); - - [Benchmark] public byte ReturnByte() => 0; - public struct Byte1 { public byte _1; } - [Benchmark] public Byte1 ReturnByte1() => new Byte1(); - public struct Byte2 { public byte _1, _2; } - [Benchmark] public Byte2 ReturnByte2() => new Byte2(); - public struct Byte3 { public byte _1, _2, _3; } - [Benchmark] public Byte3 ReturnByte3() => new Byte3(); - public struct Byte4 { public byte _1, _2, _3, _4; } - [Benchmark] public Byte4 ReturnByte4() => new Byte4(); - - [Benchmark] public short ReturnShort() => 0; - public struct Short1 { public short _1; } - [Benchmark] public Short1 ReturnShort1() => new Short1(); - public struct Short2 { public short _1, _2; } - [Benchmark] public Short2 ReturnShort2() => new Short2(); - public struct Short3 { public short _1, _2, _3; } - [Benchmark] public Short3 ReturnShort3() => new Short3(); - public struct Short4 { public short _1, _2, _3, _4; } - [Benchmark] public Short4 ReturnShort4() => new Short4(); - - [Benchmark] public int ReturnInt() => 0; - public struct Int1 { public int _1; } - [Benchmark] public Int1 ReturnInt1() => new Int1(); - public struct Int2 { public int _1, _2; } - [Benchmark] public Int2 ReturnInt2() => new Int2(); - public struct Int3 { public int _1, _2, _3; } - [Benchmark] public Int3 ReturnInt3() => new Int3(); - public struct Int4 { public int _1, _2, _3, _4; } - [Benchmark] public Int4 ReturnInt4() => new Int4(); - - [Benchmark] public long ReturnLong() => 0; - public struct Long1 { public long _1; } - [Benchmark] public Long1 ReturnLong1() => new Long1(); - public struct Long2 { public long _1, _2; } - [Benchmark] public Long2 ReturnLong2() => new Long2(); - public struct Long3 { public long _1, _2, _3; } - [Benchmark] public Long3 ReturnLong3() => new Long3(); - public struct Long4 { public long _1, _2, _3, _4; } - [Benchmark] public Long4 ReturnLong4() => new Long4(); - } - // ReSharper restore InconsistentNaming -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.Static/BenchmarkDotNet.IntegrationTests.Static.csproj b/tests/BenchmarkDotNet.IntegrationTests.Static/BenchmarkDotNet.IntegrationTests.Static.csproj index 4a52743deb..3e06189dc7 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.Static/BenchmarkDotNet.IntegrationTests.Static.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.Static/BenchmarkDotNet.IntegrationTests.Static.csproj @@ -2,7 +2,7 @@ BenchmarkDotNet.IntegrationTests.Static - net462;net7.0 + net462;net8.0 true BenchmarkDotNet.IntegrationTests.Static BenchmarkDotNet.IntegrationTests.Static @@ -14,7 +14,7 @@ - + \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.VisualBasic/BenchmarkDotNet.IntegrationTests.VisualBasic.vbproj b/tests/BenchmarkDotNet.IntegrationTests.VisualBasic/BenchmarkDotNet.IntegrationTests.VisualBasic.vbproj index 973fbb08c2..0a456372e8 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.VisualBasic/BenchmarkDotNet.IntegrationTests.VisualBasic.vbproj +++ b/tests/BenchmarkDotNet.IntegrationTests.VisualBasic/BenchmarkDotNet.IntegrationTests.VisualBasic.vbproj @@ -5,7 +5,7 @@ - net462;net7.0 + net462;net8.0 diff --git a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs index 60bdd52845..36ea5688ad 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs @@ -48,13 +48,20 @@ public AllSetupAndCleanupTest(ITestOutputHelper output) : base(output) { } private static string[] GetActualLogLines(Summary summary) => GetSingleStandardOutput(summary).Where(line => line.StartsWith(Prefix)).ToArray(); - [Fact] - public void AllSetupAndCleanupMethodRunsTest() + [Theory] + [InlineData(typeof(AllSetupAndCleanupAttributeBenchmarks))] + [InlineData(typeof(AllSetupAndCleanupAttributeBenchmarksTask))] + [InlineData(typeof(AllSetupAndCleanupAttributeBenchmarksGenericTask))] + [InlineData(typeof(AllSetupAndCleanupAttributeBenchmarksValueTask))] + [InlineData(typeof(AllSetupAndCleanupAttributeBenchmarksGenericValueTask))] + [InlineData(typeof(AllSetupAndCleanupAttributeBenchmarksValueTaskSource))] + [InlineData(typeof(AllSetupAndCleanupAttributeBenchmarksGenericValueTaskSource))] + public void AllSetupAndCleanupMethodRunsTest(Type benchmarkType) { var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); var config = CreateSimpleConfig(job: miniJob); - var summary = CanExecute(config); + var summary = CanExecute(benchmarkType, config); var actualLogLines = GetActualLogLines(summary); foreach (string line in actualLogLines) @@ -83,21 +90,7 @@ public class AllSetupAndCleanupAttributeBenchmarks public void Benchmark() => Console.WriteLine(BenchmarkCalled); } - [Fact] - public void AllSetupAndCleanupMethodRunsAsyncTest() - { - var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(job: miniJob); - - var summary = CanExecute(config); - - var actualLogLines = GetActualLogLines(summary); - foreach (string line in actualLogLines) - Output.WriteLine(line); - Assert.Equal(expectedLogLines, actualLogLines); - } - - public class AllSetupAndCleanupAttributeBenchmarksAsync + public class AllSetupAndCleanupAttributeBenchmarksTask { private int setupCounter; private int cleanupCounter; @@ -115,24 +108,10 @@ public class AllSetupAndCleanupAttributeBenchmarksAsync public Task GlobalCleanup() => Console.Out.WriteLineAsync(GlobalCleanupCalled); [Benchmark] - public Task Benchmark() => Console.Out.WriteLineAsync(BenchmarkCalled); - } - - [Fact] - public void AllSetupAndCleanupMethodRunsAsyncTaskSetupTest() - { - var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(job: miniJob); - - var summary = CanExecute(config); - - var actualLogLines = GetActualLogLines(summary); - foreach (string line in actualLogLines) - Output.WriteLine(line); - Assert.Equal(expectedLogLines, actualLogLines); + public void Benchmark() => Console.WriteLine(BenchmarkCalled); } - public class AllSetupAndCleanupAttributeBenchmarksAsyncTaskSetup + public class AllSetupAndCleanupAttributeBenchmarksGenericTask { private int setupCounter; private int cleanupCounter; @@ -144,30 +123,47 @@ public class AllSetupAndCleanupAttributeBenchmarksAsyncTaskSetup public void IterationCleanup() => Console.WriteLine(IterationCleanupCalled + " (" + ++cleanupCounter + ")"); [GlobalSetup] - public Task GlobalSetup() => Console.Out.WriteLineAsync(GlobalSetupCalled); + public async Task GlobalSetup() + { + await Console.Out.WriteLineAsync(GlobalSetupCalled); + + return 42; + } [GlobalCleanup] - public Task GlobalCleanup() => Console.Out.WriteLineAsync(GlobalCleanupCalled); + public async Task GlobalCleanup() + { + await Console.Out.WriteLineAsync(GlobalCleanupCalled); + + return 42; + } [Benchmark] public void Benchmark() => Console.WriteLine(BenchmarkCalled); } - [Fact] - public void AllSetupAndCleanupMethodRunsAsyncGenericTaskSetupTest() + public class AllSetupAndCleanupAttributeBenchmarksValueTask { - var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(job: miniJob); + private int setupCounter; + private int cleanupCounter; - var summary = CanExecute(config); + [IterationSetup] + public void IterationSetup() => Console.WriteLine(IterationSetupCalled + " (" + ++setupCounter + ")"); - var actualLogLines = GetActualLogLines(summary); - foreach (string line in actualLogLines) - Output.WriteLine(line); - Assert.Equal(expectedLogLines, actualLogLines); + [IterationCleanup] + public void IterationCleanup() => Console.WriteLine(IterationCleanupCalled + " (" + ++cleanupCounter + ")"); + + [GlobalSetup] + public ValueTask GlobalSetup() => new ValueTask(Console.Out.WriteLineAsync(GlobalSetupCalled)); + + [GlobalCleanup] + public ValueTask GlobalCleanup() => new ValueTask(Console.Out.WriteLineAsync(GlobalCleanupCalled)); + + [Benchmark] + public void Benchmark() => Console.WriteLine(BenchmarkCalled); } - public class AllSetupAndCleanupAttributeBenchmarksAsyncGenericTaskSetup + public class AllSetupAndCleanupAttributeBenchmarksGenericValueTask { private int setupCounter; private int cleanupCounter; @@ -179,7 +175,7 @@ public class AllSetupAndCleanupAttributeBenchmarksAsyncGenericTaskSetup public void IterationCleanup() => Console.WriteLine(IterationCleanupCalled + " (" + ++cleanupCounter + ")"); [GlobalSetup] - public async Task GlobalSetup() + public async ValueTask GlobalSetup() { await Console.Out.WriteLineAsync(GlobalSetupCalled); @@ -187,7 +183,7 @@ public async Task GlobalSetup() } [GlobalCleanup] - public async Task GlobalCleanup() + public async ValueTask GlobalCleanup() { await Console.Out.WriteLineAsync(GlobalCleanupCalled); @@ -198,22 +194,9 @@ public async Task GlobalCleanup() public void Benchmark() => Console.WriteLine(BenchmarkCalled); } - [Fact] - public void AllSetupAndCleanupMethodRunsAsyncValueTaskSetupTest() - { - var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(job: miniJob); - - var summary = CanExecute(config); - - var actualLogLines = GetActualLogLines(summary); - foreach (string line in actualLogLines) - Output.WriteLine(line); - Assert.Equal(expectedLogLines, actualLogLines); - } - - public class AllSetupAndCleanupAttributeBenchmarksAsyncValueTaskSetup + public class AllSetupAndCleanupAttributeBenchmarksValueTaskSource { + private readonly ValueTaskSource valueTaskSource = new(); private int setupCounter; private int cleanupCounter; @@ -224,31 +207,28 @@ public class AllSetupAndCleanupAttributeBenchmarksAsyncValueTaskSetup public void IterationCleanup() => Console.WriteLine(IterationCleanupCalled + " (" + ++cleanupCounter + ")"); [GlobalSetup] - public ValueTask GlobalSetup() => new ValueTask(Console.Out.WriteLineAsync(GlobalSetupCalled)); + public ValueTask GlobalSetup() + { + valueTaskSource.Reset(); + Console.Out.WriteLineAsync(GlobalSetupCalled).ContinueWith(_ => valueTaskSource.SetResult(42)); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } [GlobalCleanup] - public ValueTask GlobalCleanup() => new ValueTask(Console.Out.WriteLineAsync(GlobalCleanupCalled)); + public ValueTask GlobalCleanup() + { + valueTaskSource.Reset(); + Console.Out.WriteLineAsync(GlobalCleanupCalled).ContinueWith(_ => valueTaskSource.SetResult(42)); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } [Benchmark] public void Benchmark() => Console.WriteLine(BenchmarkCalled); } - [FactEnvSpecific(EnvRequirement.NonWindows)] - public void AllSetupAndCleanupMethodRunsAsyncGenericValueTaskSetupTest() - { - var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(job: miniJob); - - var summary = CanExecute(config); - - var actualLogLines = GetActualLogLines(summary); - foreach (string line in actualLogLines) - Output.WriteLine(line); - Assert.Equal(expectedLogLines, actualLogLines); - } - - public class AllSetupAndCleanupAttributeBenchmarksAsyncGenericValueTaskSetup + public class AllSetupAndCleanupAttributeBenchmarksGenericValueTaskSource { + private readonly ValueTaskSource valueTaskSource = new(); private int setupCounter; private int cleanupCounter; @@ -259,19 +239,19 @@ public class AllSetupAndCleanupAttributeBenchmarksAsyncGenericValueTaskSetup public void IterationCleanup() => Console.WriteLine(IterationCleanupCalled + " (" + ++cleanupCounter + ")"); [GlobalSetup] - public async ValueTask GlobalSetup() + public ValueTask GlobalSetup() { - await Console.Out.WriteLineAsync(GlobalSetupCalled); - - return 42; + valueTaskSource.Reset(); + Console.Out.WriteLineAsync(GlobalSetupCalled).ContinueWith(_ => valueTaskSource.SetResult(42)); + return new ValueTask(valueTaskSource, valueTaskSource.Token); } [GlobalCleanup] - public async ValueTask GlobalCleanup() + public ValueTask GlobalCleanup() { - await Console.Out.WriteLineAsync(GlobalCleanupCalled); - - return 42; + valueTaskSource.Reset(); + Console.Out.WriteLineAsync(GlobalCleanupCalled).ContinueWith(_ => valueTaskSource.SetResult(42)); + return new ValueTask(valueTaskSource, valueTaskSource.Token); } [Benchmark] diff --git a/tests/BenchmarkDotNet.IntegrationTests/AppBundle/test-main.js b/tests/BenchmarkDotNet.IntegrationTests/AppBundle/test-main.js new file mode 100644 index 0000000000..c9f705d547 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/AppBundle/test-main.js @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +import { dotnet } from './_framework/dotnet.js' + +await dotnet + .withDiagnosticTracing(false) + .withApplicationArguments(...arguments) + .run() diff --git a/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs index 8c5453a247..021ff5d96d 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs @@ -1,13 +1,13 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Numerics; -using System.Threading.Tasks; -using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.InProcess.Emit; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Numerics; +using System.Threading.Tasks; using Xunit; using Xunit.Abstractions; @@ -24,7 +24,7 @@ public static IEnumerable GetToolchains() public ArgumentsTests(ITestOutputHelper output) : base(output) { } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ArgumentsArePassedToBenchmarks(IToolchain toolchain) => CanExecute(toolchain); public class WithArguments @@ -61,7 +61,7 @@ public ValueTask SimpleValueTaskAsync(bool boolean, int number) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ArgumentsFromSourceArePassedToBenchmarks(IToolchain toolchain) => CanExecute(toolchain); public class WithArgumentsSource @@ -81,7 +81,94 @@ public IEnumerable ArgumentsProvider() } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + public void ArgumentsFromSourceInAnotherClassArePassedToBenchmarks(IToolchain toolchain) => CanExecute(toolchain); + + public class WithArgumentsSourceInAnotherClass + { + [Benchmark] + [ArgumentsSource(typeof(ExternalClassWithArgumentsSource), nameof(ExternalClassWithArgumentsSource.OnePrimitiveType))] + public void OnePrimitiveType(int number) + { + if (number % 2 != 1) + throw new InvalidOperationException("Incorrect values were passed"); + } + + [Benchmark] + [ArgumentsSource(typeof(ExternalClassWithArgumentsSource), nameof(ExternalClassWithArgumentsSource.TwoPrimitiveTypes))] + public void TwoPrimitiveTypes(bool boolean, int number) + { + if (boolean && number != 1 || !boolean && number != 2) + throw new InvalidOperationException("Incorrect values were passed"); + } + + [Benchmark] + [ArgumentsSource(typeof(ExternalClassWithArgumentsSource), nameof(ExternalClassWithArgumentsSource.OneNonPrimitiveType))] + public void OneNonPrimitiveType(Version version) + { + int[] versionNumbers = { version.Major, version.Minor, version.MinorRevision, version.Build }; + if (versionNumbers.Distinct().Count() != 4) + throw new InvalidOperationException("Incorrect values were passed"); + } + + [Benchmark] + [ArgumentsSource(typeof(ExternalClassWithArgumentsSource), nameof(ExternalClassWithArgumentsSource.TwoNonPrimitiveTypes))] + public void TwoNonPrimitiveTypes(Version version, DateTime dateTime) + { + int[] versionNumbers = { version.Major, version.Minor, version.MinorRevision, version.Build }; + if (versionNumbers.Distinct().Count() != 4) + throw new InvalidOperationException("Incorrect values were passed"); + + if (dateTime.Month != dateTime.Day) + throw new InvalidOperationException("Incorrect values were passed"); + } + + [Benchmark] + [ArgumentsSource(typeof(ExternalClassWithArgumentsSource), nameof(ExternalClassWithArgumentsSource.OnePrimitiveAndOneNonPrimitive))] + public void OnePrimitiveAndOneNonPrimitive(Version version, int number) + { + int[] versionNumbers = { version.Major, version.Minor, version.MinorRevision, version.Build }; + if (versionNumbers.Distinct().Count() != 4) + throw new InvalidOperationException("Incorrect values were passed"); + + if (number != version.Major) + throw new InvalidOperationException("Incorrect values were passed"); + } + } + public static class ExternalClassWithArgumentsSource + { + public static IEnumerable OnePrimitiveType() + { + yield return 3; + yield return 5; + } + + public static IEnumerable TwoPrimitiveTypes() + { + yield return new object[] { true, 1 }; + yield return new object[] { false, 2 }; + } + + public static IEnumerable OneNonPrimitiveType() + { + yield return new Version(1, 2, 3, 4); + yield return new Version(5, 6, 7, 8); + } + + public static IEnumerable TwoNonPrimitiveTypes() + { + yield return new object[] { new Version(1, 2, 3, 4), new DateTime(2011, 11, 11) }; + yield return new object[] { new Version(5, 6, 7, 8), new DateTime(2002, 02, 02) }; + } + + public static IEnumerable OnePrimitiveAndOneNonPrimitive() + { + yield return new object[] { new Version(1, 2, 3, 4), 1 }; + yield return new object[] { new Version(5, 6, 7, 8), 5 }; + } + } + + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ArgumentsCanBePassedByReferenceToBenchmark(IToolchain toolchain) => CanExecute(toolchain); public class WithRefArguments @@ -96,7 +183,7 @@ public void Simple(ref bool boolean, ref int number) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ArgumentsCanBePassedByReadonlyReferenceToBenchmark(IToolchain toolchain) => CanExecute(toolchain); public class WithInArguments @@ -111,7 +198,7 @@ public void Simple(in bool boolean, in int number) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void NonCompileTimeConstantsCanBeReturnedFromSource(IToolchain toolchain) => CanExecute(toolchain); public class WithComplexTypesReturnedFromSources @@ -168,7 +255,7 @@ public struct SomeStruct } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ArrayCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithArray @@ -186,7 +273,7 @@ public void AcceptingArray(int[] array) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void IEnumerableCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithIEnumerable @@ -211,7 +298,7 @@ public IEnumerable Sources() public void Any(string name, IEnumerable source) => source.Any(); } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void JaggedArrayCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithJaggedArray @@ -247,7 +334,7 @@ public IEnumerable CreateMatrix() } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void GenericTypeCanBePassedByRefAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithGenericByRef @@ -283,7 +370,7 @@ public IEnumerable GetInputData() } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void AnArrayOfTypeWithNoParameterlessCtorCanBePassedAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithArrayOfStringAsArgument @@ -300,7 +387,7 @@ public void TypeReflectionArrayGetType(object anArray) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void AnArrayCanBePassedToBenchmarkAsSpan(IToolchain toolchain) => CanExecute(toolchain); public class WithArrayToSpan @@ -320,7 +407,7 @@ public void AcceptsSpan(Span span) [TheoryEnvSpecific("The implicit cast operator is available only in .NET Core 2.1+ (See https://github.com/dotnet/corefx/issues/30121 for more)", EnvRequirement.DotNetCoreOnly)] - [MemberData(nameof(GetToolchains))] + [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void StringCanBePassedToBenchmarkAsReadOnlySpan(IToolchain toolchain) => CanExecute(toolchain); public class WithStringToReadOnlySpan @@ -338,7 +425,7 @@ public void AcceptsReadOnlySpan(ReadOnlySpan notString) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void AnArrayOfStringsCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); @@ -358,7 +445,7 @@ public void TypeReflectionArrayGetType(string[] array) } } - [Theory, MemberData(nameof(GetToolchains))] // make sure BDN mimics xunit's MemberData behaviour + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] // make sure BDN mimics xunit's MemberData behaviour public void AnIEnumerableOfArrayOfObjectsCanBeUsedAsArgumentForBenchmarkAcceptingSingleArgument(IToolchain toolchain) => CanExecute(toolchain); @@ -378,7 +465,7 @@ public void SingleArgument(bool boolean) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void BenchmarkCanAcceptFewArrays(IToolchain toolchain) => CanExecute(toolchain); public class FewArrays @@ -407,7 +494,7 @@ public void AcceptsArrays(int[] even, int[] notEven) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void VeryBigIntegersAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithVeryBigInteger @@ -428,7 +515,7 @@ public void Method(BigInteger passed) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void SpecialDoubleValuesAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithSpecialDoubleValues @@ -473,7 +560,7 @@ public void Method(double passed, string name) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void SpecialFloatValuesAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithSpecialFloatValues @@ -518,7 +605,7 @@ public void Method(float passed, string name) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void SpecialDecimalValuesAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithSpecialDecimalValues @@ -547,7 +634,7 @@ public void Method(decimal passed, string name) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void DateTimeCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithDateTime @@ -568,7 +655,7 @@ public void Test(DateTime passed) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void CustomTypeThatAlsoExistsInTheSystemNamespaceAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class CustomTypeThatAlsoExistsInTheSystemNamespace @@ -589,7 +676,7 @@ public void Test(Action passed) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void EnumFlagsAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithEnumFlags @@ -626,7 +713,7 @@ public void Test(LongFlagEnum passedLongFlagEnum, ByteFlagEnum passedByteFlagEnu } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void UndefinedEnumValuesAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithUndefinedEnumValue @@ -652,7 +739,7 @@ public void Test(SomeEnum defined, SomeEnum undefined, SomeEnum undefinedNegativ } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void StaticMethodsAndPropertiesCanBeUsedAsSources_EnumerableOfObjects(IToolchain toolchain) => CanExecute(toolchain); @@ -700,7 +787,7 @@ public void TestProperty(int argument) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void StaticMethodsAndPropertiesCanBeUsedAsSources_EnumerableOfArrayOfObjects(IToolchain toolchain) => CanExecute(toolchain); @@ -747,7 +834,64 @@ public void TestProperty(int argument) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + public void MethodsAndPropertiesFromAnotherClassCanBeUsedAsSources(IToolchain toolchain) + => CanExecute(toolchain); + + public class ParamsSourcePointingToAnotherClass + { + [ParamsSource(typeof(ExternalClassWithParamsSource), nameof(ExternalClassWithParamsSource.PrimitiveTypeMethod))] + public int ParamOne { get; set; } + + [ParamsSource(typeof(ExternalClassWithParamsSource), nameof(ExternalClassWithParamsSource.PrimitiveTypeProperty))] + public int ParamTwo { get; set; } + + [ParamsSource(typeof(ExternalClassWithParamsSource), nameof(ExternalClassWithParamsSource.NonPrimitiveTypeMethod))] + public Version ParamThree { get; set; } + + [ParamsSource(typeof(ExternalClassWithParamsSource), nameof(ExternalClassWithParamsSource.NonPrimitiveTypeProperty))] + public Version ParamFour { get; set; } + + [Benchmark] + public void Test() + { + if (ParamOne != 123) + throw new ArgumentException("The ParamOne value is incorrect!"); + if (ParamTwo != 456) + throw new ArgumentException("The ParamTwo value is incorrect!"); + if (ParamThree != new Version(1, 2, 3, 4)) + throw new ArgumentException("The ParamThree value is incorrect!"); + if (ParamFour != new Version(5, 6, 7, 8)) + throw new ArgumentException("The ParamFour value is incorrect!"); + } + } + public static class ExternalClassWithParamsSource + { + public static IEnumerable PrimitiveTypeMethod() + { + yield return 123; + } + public static IEnumerable PrimitiveTypeProperty + { + get + { + yield return 456; + } + } + public static IEnumerable NonPrimitiveTypeMethod() + { + yield return new Version(1, 2, 3, 4); + } + public static IEnumerable NonPrimitiveTypeProperty + { + get + { + yield return new Version(5, 6, 7, 8); + } + } + } + + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void VeryLongStringsAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithVeryLongString @@ -771,7 +915,7 @@ public void Test(string first, string second) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ComplexStringPattersAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class Perf_Regex_Industry_RustLang_Sherlock diff --git a/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs b/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs index 8eb149b5dc..a2158362ef 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs @@ -1,10 +1,44 @@ -using System.Threading.Tasks; +using System; +using System.Threading.Tasks; +using System.Threading.Tasks.Sources; using BenchmarkDotNet.Attributes; using Xunit; using Xunit.Abstractions; namespace BenchmarkDotNet.IntegrationTests { + internal class ValueTaskSource : IValueTaskSource, IValueTaskSource + { + private ManualResetValueTaskSourceCore _core; + + T IValueTaskSource.GetResult(short token) => _core.GetResult(token); + void IValueTaskSource.GetResult(short token) => _core.GetResult(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _core.GetStatus(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _core.GetStatus(token); + void IValueTaskSource.OnCompleted(Action continuation, object state, short token, ValueTaskSourceOnCompletedFlags flags) => _core.OnCompleted(continuation, state, token, flags); + void IValueTaskSource.OnCompleted(Action continuation, object state, short token, ValueTaskSourceOnCompletedFlags flags) => _core.OnCompleted(continuation, state, token, flags); + public void Reset() => _core.Reset(); + public short Token => _core.Version; + public void SetResult(T result) => _core.SetResult(result); + } + + // This is used to test the case of ValueTaskAwaiter.IsCompleted returns false, then OnCompleted invokes the callback immediately because it happened to complete between the 2 calls. + internal class ValueTaskSourceCallbackOnly : IValueTaskSource, IValueTaskSource + { + private ManualResetValueTaskSourceCore _core; + + T IValueTaskSource.GetResult(short token) => _core.GetResult(token); + void IValueTaskSource.GetResult(short token) => _core.GetResult(token); + // Always return pending state so OnCompleted will be called. + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => ValueTaskSourceStatus.Pending; + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => ValueTaskSourceStatus.Pending; + void IValueTaskSource.OnCompleted(Action continuation, object state, short token, ValueTaskSourceOnCompletedFlags flags) => _core.OnCompleted(continuation, state, token, flags); + void IValueTaskSource.OnCompleted(Action continuation, object state, short token, ValueTaskSourceOnCompletedFlags flags) => _core.OnCompleted(continuation, state, token, flags); + public void Reset() => _core.Reset(); + public short Token => _core.Version; + public void SetResult(T result) => _core.SetResult(result); + } + public class AsyncBenchmarksTests : BenchmarkTestExecutor { public AsyncBenchmarksTests(ITestOutputHelper output) : base(output) { } @@ -24,8 +58,13 @@ public void TaskReturningMethodsAreAwaited() } } + [Fact] + public void TaskReturningMethodsAreAwaited_AlreadyComplete() => CanExecute(); + public class TaskDelayMethods { + private readonly ValueTaskSource valueTaskSource = new(); + private const int MillisecondsDelay = 100; internal const double NanosecondsDelay = MillisecondsDelay * 1e+6; @@ -39,6 +78,17 @@ public class TaskDelayMethods [Benchmark] public ValueTask ReturningValueTask() => new ValueTask(Task.Delay(MillisecondsDelay)); + [Benchmark] + public ValueTask ReturningValueTaskBackByIValueTaskSource() + { + valueTaskSource.Reset(); + Task.Delay(MillisecondsDelay).ContinueWith(_ => + { + valueTaskSource.SetResult(default); + }); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + [Benchmark] public async Task Awaiting() => await Task.Delay(MillisecondsDelay); @@ -47,6 +97,70 @@ public class TaskDelayMethods [Benchmark] public ValueTask ReturningGenericValueTask() => new ValueTask(ReturningGenericTask()); + + [Benchmark] + public ValueTask ReturningGenericValueTaskBackByIValueTaskSource() + { + valueTaskSource.Reset(); + Task.Delay(MillisecondsDelay).ContinueWith(_ => + { + valueTaskSource.SetResult(default); + }); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + } + + public class TaskImmediateMethods + { + private readonly ValueTaskSource valueTaskSource = new(); + private readonly ValueTaskSourceCallbackOnly valueTaskSourceCallbackOnly = new(); + + [Benchmark] + public Task ReturningTask() => Task.CompletedTask; + + [Benchmark] + public ValueTask ReturningValueTask() => new ValueTask(); + + [Benchmark] + public ValueTask ReturningValueTaskBackByIValueTaskSource() + { + valueTaskSource.Reset(); + valueTaskSource.SetResult(default); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + + [Benchmark] + public ValueTask ReturningValueTaskBackByIValueTaskSource_ImmediateCallback() + { + valueTaskSourceCallbackOnly.Reset(); + valueTaskSourceCallbackOnly.SetResult(default); + return new ValueTask(valueTaskSourceCallbackOnly, valueTaskSourceCallbackOnly.Token); + } + + [Benchmark] + public async Task Awaiting() => await Task.CompletedTask; + + [Benchmark] + public Task ReturningGenericTask() => ReturningTask().ContinueWith(_ => default(int)); + + [Benchmark] + public ValueTask ReturningGenericValueTask() => new ValueTask(ReturningGenericTask()); + + [Benchmark] + public ValueTask ReturningGenericValueTaskBackByIValueTaskSource() + { + valueTaskSource.Reset(); + valueTaskSource.SetResult(default); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + + [Benchmark] + public ValueTask ReturningGenericValueTaskBackByIValueTaskSource_ImmediateCallback() + { + valueTaskSourceCallbackOnly.Reset(); + valueTaskSourceCallbackOnly.SetResult(default); + return new ValueTask(valueTaskSourceCallbackOnly, valueTaskSourceCallbackOnly.Token); + } } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj index 378822308f..46e9b0edac 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj @@ -2,8 +2,8 @@ BenchmarkDotNet.IntegrationTests - net462;net7.0 - net7.0 + net462;net8.0 + net8.0 true BenchmarkDotNet.IntegrationTests BenchmarkDotNet.IntegrationTests @@ -18,6 +18,8 @@ Always + + @@ -29,25 +31,29 @@ + - - - - + + + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + - + - - + + @@ -55,6 +61,7 @@ + diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs index e9a95fce81..622aee3ffd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs @@ -66,7 +66,7 @@ public void WhenInvalidTypeIsProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAre .Run(new[] { "--filter", "*" }, config); Assert.Empty(summaries); - Assert.Contains("Type BenchmarkDotNet.IntegrationTests.ClassC is invalid.", logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(ClassC)), logger.GetLog()); } [Fact] @@ -80,7 +80,7 @@ public void WhenNoTypesAreProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAreExe .Run(new[] { "--filter", "*" }, config); Assert.Empty(summaries); - Assert.Contains("No benchmarks to choose from. Make sure you provided public non-sealed non-static types with public [Benchmark] methods.", logger.GetLog()); + Assert.Contains("No benchmarks were found.", logger.GetLog()); } [Fact] @@ -369,6 +369,11 @@ public IReadOnlyList AskUser(IReadOnlyList allTypes, ILogger logger) return returnValue; } } + + private string GetValidationErrorForType(Type type) + { + return $"No [Benchmark] attribute found on '{type.Name}' benchmark case."; + } } } @@ -424,6 +429,8 @@ public override void ExportToLog(Summary summary, ILogger logger) exported = true; } } + + } namespace BenchmarkDotNet.NOTIntegrationTests diff --git a/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs b/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs index b495489d78..101fa50139 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs @@ -1,6 +1,7 @@ using System; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -19,7 +20,7 @@ public void WhenBuildTakesMoreTimeThanTheTimeoutTheBuildIsCancelled() { if (!RuntimeInformation.Is64BitPlatform()) // NativeAOT does not support 32bit yet return; - if (RuntimeInformation.IsMacOS()) + if (OsDetector.IsMacOS()) return; // currently not supported // we use NativeAOT on purpose because it takes a LOT of time to build it @@ -29,10 +30,10 @@ public void WhenBuildTakesMoreTimeThanTheTimeoutTheBuildIsCancelled() var config = ManualConfig.CreateEmpty() .WithBuildTimeout(timeout) .AddJob(Job.Dry - .WithRuntime(NativeAotRuntime.Net70) + .WithRuntime(NativeAotRuntime.Net80) .WithToolchain(NativeAotToolchain.CreateBuilder() - .UseNuGet("7.0.0", "https://api.nuget.org/v3/index.json") - .TargetFrameworkMoniker("net7.0") + .UseNuGet("8.0.0", "https://api.nuget.org/v3/index.json") + .TargetFrameworkMoniker("net8.0") .ToToolchain())); var summary = CanExecute(config, fullValidation: false); diff --git a/tests/BenchmarkDotNet.IntegrationTests/ContinuousIntegration.cs b/tests/BenchmarkDotNet.IntegrationTests/ContinuousIntegration.cs index 0f112aa328..8b8710efe5 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ContinuousIntegration.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ContinuousIntegration.cs @@ -1,5 +1,6 @@ using BenchmarkDotNet.Portability; using System; +using BenchmarkDotNet.Detectors; namespace BenchmarkDotNet.IntegrationTests { @@ -8,7 +9,7 @@ internal static class ContinuousIntegration private static bool IsGitHubActions() => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GITHUB_ACTION")); internal static bool IsGitHubActionsOnWindows() - => RuntimeInformation.IsWindows() && IsGitHubActions(); + => OsDetector.IsWindows() && IsGitHubActions(); internal static bool IsLocalRun() => !IsGitHubActions(); } diff --git a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs index 43a7bac9ee..f6d0a10f83 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs @@ -74,8 +74,8 @@ public RunResults Run() return new RunResults( new List { - new (1, IterationMode.Overhead, IterationStage.Actual, 1, 1, 1), - new (1, IterationMode.Workload, IterationStage.Actual, 1, 1, 1) + new(1, IterationMode.Overhead, IterationStage.Actual, 1, 1, 1), + new(1, IterationMode.Workload, IterationStage.Actual, 1, 1, 1) }, OutlierMode.DontRemove, default, diff --git a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs index 49bd585709..488d76ca31 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Disassemblers; using BenchmarkDotNet.Engines; @@ -34,14 +35,14 @@ public static IEnumerable GetAllJits() { if (RuntimeInformation.GetCurrentPlatform() is Platform.X86 or Platform.X64) { - yield return new object[] { Jit.RyuJit, Platform.X64, CoreRuntime.Core70 }; // .NET Core x64 + yield return new object[] { Jit.RyuJit, Platform.X64, CoreRuntime.Core80 }; // .NET Core x64 } - else if (RuntimeInformation.GetCurrentPlatform() is Platform.Arm64 && RuntimeInformation.IsLinux()) + else if (RuntimeInformation.GetCurrentPlatform() is Platform.Arm64 && OsDetector.IsLinux()) { - yield return new object[] { Jit.RyuJit, Platform.Arm64, CoreRuntime.Core70 }; // .NET Core arm64 + yield return new object[] { Jit.RyuJit, Platform.Arm64, CoreRuntime.Core80 }; // .NET Core arm64 } } - if (RuntimeInformation.IsMacOS()) + if (OsDetector.IsMacOS()) { // This scope of tests is not supported on macOS // However, when the MemberData method provides no data, xUnit throws an "No data found" InvalidOperationException @@ -87,41 +88,49 @@ public void Recursive() } [Theory] - [MemberData(nameof(GetAllJits))] + [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, Runtime runtime) { - if (RuntimeInformation.IsMacOS()) return; // currently not supported + if (OsDetector.IsMacOS()) return; // currently not supported + var printSource = IsPrintSourceSupported(platform); var disassemblyDiagnoser = new DisassemblyDiagnoser( - new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); + new DisassemblyDiagnoserConfig(printSource: printSource, maxDepth: 3)); CanExecute(CreateConfig(jit, platform, runtime, disassemblyDiagnoser, RunStrategy.ColdStart)); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Benchmark)}(Int32)"); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Benchmark)}(Boolean)"); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Static)}()"); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Instance)}()"); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Recursive)}()"); + DisassemblyResult result = disassemblyDiagnoser.Results.Single().Value; + + Assert.Empty(result.Errors); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Benchmark)}(Int32)"); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Benchmark)}(Boolean)"); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Static)}()"); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Instance)}()"); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Recursive)}()"); } [Theory] - [MemberData(nameof(GetAllJits))] + [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, Runtime runtime) { - if (RuntimeInformation.IsMacOS()) return; // currently not supported + if (OsDetector.IsMacOS()) return; // currently not supported + var printSource = IsPrintSourceSupported(platform); var disassemblyDiagnoser = new DisassemblyDiagnoser( - new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 1, filters: new[] { "*WithCalls*" })); + new DisassemblyDiagnoserConfig(printSource: printSource, maxDepth: 1, filters: new[] { "*WithCalls*" })); CanExecute(CreateConfig(jit, platform, runtime, disassemblyDiagnoser, RunStrategy.ColdStart)); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Benchmark)}(Int32)"); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Benchmark)}(Boolean)"); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Static)}()"); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Instance)}()"); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Recursive)}()"); + DisassemblyResult result = disassemblyDiagnoser.Results.Single().Value; + + Assert.Empty(result.Errors); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Benchmark)}(Int32)"); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Benchmark)}(Boolean)"); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Static)}()"); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Instance)}()"); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Recursive)}()"); } public class Generic where T : new() @@ -131,19 +140,21 @@ public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, } [Theory] - [MemberData(nameof(GetAllJits))] + [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleGenericTypes(Jit jit, Platform platform, Runtime runtime) { - if (RuntimeInformation.IsMacOS()) return; // currently not supported + if (OsDetector.IsMacOS()) return; // currently not supported + var printSource = IsPrintSourceSupported(platform); var disassemblyDiagnoser = new DisassemblyDiagnoser( - new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); + new DisassemblyDiagnoserConfig(printSource: printSource, maxDepth: 3)); CanExecute>(CreateConfig(jit, platform, runtime, disassemblyDiagnoser, RunStrategy.Monitoring)); var result = disassemblyDiagnoser.Results.Values.Single(); + Assert.Empty(result.Errors); Assert.Contains(result.Methods, method => method.Maps.Any(map => map.SourceCodes.OfType().Any())); } @@ -153,19 +164,21 @@ [Benchmark] public void JustReturn() { } } [Theory] - [MemberData(nameof(GetAllJits))] + [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleInlinableBenchmarks(Jit jit, Platform platform, Runtime runtime) { - if (RuntimeInformation.IsMacOS()) return; // currently not supported + if (OsDetector.IsMacOS()) return; // currently not supported + var printSource = IsPrintSourceSupported(platform); var disassemblyDiagnoser = new DisassemblyDiagnoser( - new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); + new DisassemblyDiagnoserConfig(printSource: printSource, maxDepth: 3)); CanExecute(CreateConfig(jit, platform, runtime, disassemblyDiagnoser, RunStrategy.Monitoring)); var disassemblyResult = disassemblyDiagnoser.Results.Values.Single(result => result.Methods.Count(method => method.Name.Contains(nameof(WithInlineable.JustReturn))) == 1); + Assert.Empty(disassemblyResult.Errors); Assert.Contains(disassemblyResult.Methods, method => method.Maps.Any(map => map.SourceCodes.OfType().All(asm => asm.ToString().Contains("ret")))); } @@ -180,12 +193,13 @@ private IConfig CreateConfig(Jit jit, Platform platform, Runtime runtime, IDiagn .AddDiagnoser(disassemblyDiagnoser) .AddLogger(new OutputLogger(Output)); - private void AssertDisassembled(DisassemblyDiagnoser diagnoser, string methodSignature) + private void AssertDisassemblyResult(DisassemblyResult result, string methodSignature) { - DisassemblyResult result = diagnoser.Results.Single().Value; - Assert.Contains(methodSignature, result.Methods.Select(m => m.Name.Split('.').Last()).ToArray()); Assert.Contains(result.Methods.Single(m => m.Name.EndsWith(methodSignature)).Maps, map => map.SourceCodes.Any()); } + + private static bool IsPrintSourceSupported(Platform platform) + => platform != Platform.X86; // Workaround for https://github.com/dotnet/BenchmarkDotNet/issues/2789 } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/DotMemoryTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DotMemoryTests.cs new file mode 100644 index 0000000000..2f21fb9f3c --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/DotMemoryTests.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Diagnostics.dotMemory; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.IntegrationTests +{ + public class DotMemoryTests : BenchmarkTestExecutor + { + public DotMemoryTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void DotMemorySmokeTest() + { + if (!OsDetector.IsWindows() && RuntimeInformation.IsMono) + { + Output.WriteLine("Skip Mono on non-Windows"); + return; + } + + var config = new ManualConfig().AddJob( + Job.Dry.WithId("ExternalProcess"), + Job.Dry.WithToolchain(InProcessEmitToolchain.Instance).WithId("InProcess") + ); + string snapshotDirectory = Path.Combine(Directory.GetCurrentDirectory(), "BenchmarkDotNet.Artifacts", "snapshots"); + if (Directory.Exists(snapshotDirectory)) + Directory.Delete(snapshotDirectory, true); + + CanExecute(config); + + Output.WriteLine("---------------------------------------------"); + Output.WriteLine("SnapshotDirectory:" + snapshotDirectory); + var snapshots = Directory.EnumerateFiles(snapshotDirectory) + .Where(filePath => Path.GetExtension(filePath).Equals(".dmw", StringComparison.OrdinalIgnoreCase)) + .Select(Path.GetFileName) + .OrderBy(fileName => fileName) + .ToList(); + Output.WriteLine("Snapshots:"); + foreach (string snapshot in snapshots) + Output.WriteLine("* " + snapshot); + Assert.Equal(4, snapshots.Count); + } + + [DotMemoryDiagnoser] + public class Benchmarks + { + [Benchmark] + public int Foo0() + { + var list = new List(); + for (int i = 0; i < 1000; i++) + list.Add(new object()); + return list.Count; + } + + [Benchmark] + public int Foo1() + { + var list = new List(); + for (int i = 0; i < 1000; i++) + list.Add(new object()); + return list.Count; + } + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/DotTraceTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DotTraceTests.cs index 80f11519e8..3c40ee31a0 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DotTraceTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DotTraceTests.cs @@ -4,6 +4,7 @@ using System.Linq; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnostics.dotTrace; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -20,7 +21,7 @@ public DotTraceTests(ITestOutputHelper output) : base(output) { } [Fact] public void DotTraceSmokeTest() { - if (!RuntimeInformation.IsWindows() && RuntimeInformation.IsMono) + if (!OsDetector.IsWindows() && RuntimeInformation.IsMono) { Output.WriteLine("Skip Mono on non-Windows"); return; diff --git a/tests/BenchmarkDotNet.IntegrationTests/DryRunTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DryRunTests.cs index b9ca511236..479581c815 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DryRunTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DryRunTests.cs @@ -8,12 +8,10 @@ namespace BenchmarkDotNet.IntegrationTests { - public class DryRunTests : BenchmarkTestExecutor + public class DryRunTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) { - public DryRunTests(ITestOutputHelper output) : base(output) { } - [Fact] - public void WelchTTest() => CanExecute(); + public void BenchWithStatTest() => CanExecute(); [DryJob, StatisticalTestColumn] public class WelchTTestBench diff --git a/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs index 4cede6835b..d188be01d4 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs @@ -23,8 +23,10 @@ public class EventProcessorTests public void WhenUsingEventProcessorAndNoBenchmarks() { var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassEmpty) }); - Assert.Single(events); + Assert.Equal(3, events.Count); Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); + Assert.Equal(nameof(EventProcessor.OnValidationError), events[1].EventType); + Assert.Equal(nameof(EventProcessor.OnEndValidationStage), events[2].EventType); } [Fact] @@ -134,12 +136,13 @@ public void WhenUsingEventProcessorWithUnsupportedBenchmark() var toolchain = new AllUnsupportedToolchain(); var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassA) }, toolchain: toolchain); - Assert.Equal(3, events.Count); + Assert.Equal(4, events.Count); Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); Assert.Equal(nameof(EventProcessor.OnValidationError), events[1].EventType); Assert.Equal(typeof(ClassA).GetMethod(nameof(ClassA.Method1)), (events[1].Args[0] as ValidationError).BenchmarkCase.Descriptor.WorkloadMethod); Assert.Equal(nameof(EventProcessor.OnValidationError), events[2].EventType); Assert.Equal(typeof(ClassA).GetMethod(nameof(ClassA.Method2)), (events[2].Args[0] as ValidationError).BenchmarkCase.Descriptor.WorkloadMethod); + Assert.Equal(nameof(EventProcessor.OnEndValidationStage), events[3].EventType); } [Fact] diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs index cb034920a1..4b9ea2005b 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs @@ -10,6 +10,7 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.InProcess.Emit; @@ -65,13 +66,16 @@ private void DiffEmit(Summary summary) { // .Net Core does not support assembly saving so far // SEE https://github.com/dotnet/corefx/issues/4491 + // TODO: Use new PersistedAssemblyBuilder when BDN and tests are updated to net9.0 or newer. if (!Portability.RuntimeInformation.IsFullFramework) return; - var caseName = summary.BenchmarksCases.First().Job.ToString(); + var benchmarkCase = summary.BenchmarksCases.First(); + // The benchmark config built jobs with 2 toolchains, 1 InProcessEmit and 1 Roslyn, + // so we need to subtract 1 from the partition counter to obtain the emit output. NaiveRunnableEmitDiff.RunDiff( - $@"{caseName}.exe", - $@"{caseName}Emitted.dll", + $@"{BuildPartition.GetProgramName(benchmarkCase, BuildPartition.s_partitionCounter)}.exe", + $@"{BuildPartition.GetProgramName(benchmarkCase, BuildPartition.s_partitionCounter - 1)}Emitted.dll", ConsoleLogger.Default); } @@ -153,6 +157,13 @@ public async Task InvokeOnceTaskAsync() Interlocked.Increment(ref Counter); } + [Benchmark] + public async ValueTask InvokeOnceValueTaskAsync() + { + await Task.Yield(); + Interlocked.Increment(ref Counter); + } + [Benchmark] public string InvokeOnceRefType() { @@ -195,6 +206,13 @@ public static async Task InvokeOnceStaticTaskAsync() Interlocked.Increment(ref Counter); } + [Benchmark] + public static async ValueTask InvokeOnceStaticValueTaskAsync() + { + await Task.Yield(); + Interlocked.Increment(ref Counter); + } + [Benchmark] public static string InvokeOnceStaticRefType() { @@ -225,35 +243,146 @@ public static ValueTask InvokeOnceStaticValueTaskOfT() } } - [Fact] - public void InProcessEmitToolchainSupportsIterationSetupAndCleanup() + [Theory] + [InlineData(typeof(IterationSetupCleanup))] + [InlineData(typeof(GlobalSetupCleanupTask))] + [InlineData(typeof(GlobalSetupCleanupValueTask))] + [InlineData(typeof(GlobalSetupCleanupValueTaskSource))] + public void InProcessEmitToolchainSupportsSetupAndCleanup(Type benchmarkType) { var logger = new OutputLogger(Output); var config = CreateInProcessConfig(logger); - WithIterationSetupAndCleanup.SetupCounter = 0; - WithIterationSetupAndCleanup.BenchmarkCounter = 0; - WithIterationSetupAndCleanup.CleanupCounter = 0; + Counters.SetupCounter = 0; + Counters.BenchmarkCounter = 0; + Counters.CleanupCounter = 0; - var summary = CanExecute(config); + var summary = CanExecute(benchmarkType, config); - Assert.Equal(1, WithIterationSetupAndCleanup.SetupCounter); - Assert.Equal(16, WithIterationSetupAndCleanup.BenchmarkCounter); - Assert.Equal(1, WithIterationSetupAndCleanup.CleanupCounter); + Assert.Equal(1, Counters.SetupCounter); + Assert.Equal(16, Counters.BenchmarkCounter); + Assert.Equal(1, Counters.CleanupCounter); } - public class WithIterationSetupAndCleanup + private static class Counters { public static int SetupCounter, BenchmarkCounter, CleanupCounter; + } + public class IterationSetupCleanup + { [IterationSetup] - public void Setup() => Interlocked.Increment(ref SetupCounter); + public void Setup() => Interlocked.Increment(ref Counters.SetupCounter); [Benchmark] - public void Benchmark() => Interlocked.Increment(ref BenchmarkCounter); + public void Benchmark() => Interlocked.Increment(ref Counters.BenchmarkCounter); [IterationCleanup] - public void Cleanup() => Interlocked.Increment(ref CleanupCounter); + public void Cleanup() => Interlocked.Increment(ref Counters.CleanupCounter); + } + + public class GlobalSetupCleanupTask + { + [GlobalSetup] + public static async Task GlobalSetup() + { + await Task.Yield(); + Interlocked.Increment(ref Counters.SetupCounter); + } + + [GlobalCleanup] + public async Task GlobalCleanup() + { + await Task.Yield(); + Interlocked.Increment(ref Counters.CleanupCounter); + return 42; + } + + [Benchmark] + public void InvokeOnceVoid() + { + Interlocked.Increment(ref Counters.BenchmarkCounter); + } + } + + public class GlobalSetupCleanupValueTask + { + [GlobalSetup] + public static async ValueTask GlobalSetup() + { + await Task.Yield(); + Interlocked.Increment(ref Counters.SetupCounter); + } + + [GlobalCleanup] + public async ValueTask GlobalCleanup() + { + await Task.Yield(); + Interlocked.Increment(ref Counters.CleanupCounter); + return 42; + } + + [Benchmark] + public void InvokeOnceVoid() + { + Interlocked.Increment(ref Counters.BenchmarkCounter); + } + } + + public class GlobalSetupCleanupValueTaskSource + { + private readonly static ValueTaskSource valueTaskSource = new(); + + [GlobalSetup] + public static ValueTask GlobalSetup() + { + valueTaskSource.Reset(); + Task.Delay(1).ContinueWith(_ => + { + Interlocked.Increment(ref Counters.SetupCounter); + valueTaskSource.SetResult(42); + }); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + + [GlobalCleanup] + public ValueTask GlobalCleanup() + { + valueTaskSource.Reset(); + Task.Delay(1).ContinueWith(_ => + { + Interlocked.Increment(ref Counters.CleanupCounter); + valueTaskSource.SetResult(42); + }); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + + [Benchmark] + public void InvokeOnceVoid() + { + Interlocked.Increment(ref Counters.BenchmarkCounter); + } + } + + +#if NET8_0_OR_GREATER + [Fact] + public void ParamsSupportRequiredProperty() + { + var config = CreateInProcessConfig(null); + CanExecute(config); + } + + public class ParamsTestRequiredProperty + { + private const string Expected = "a"; + + [Params(Expected)] + public required string ParamProperty { get; set; } + + [Benchmark] + public void Benchmark() => Assert.Equal(Expected, ParamProperty); } +#endif } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs index 619eaa5740..97c467a2bd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs @@ -273,5 +273,26 @@ public ValueTask InvokeOnceValueTaskOfT() return new ValueTask(DecimalResult); } } + + +#if NET8_0_OR_GREATER + [Fact] + public void ParamsSupportRequiredProperty() + { + var config = CreateInProcessConfig(); + CanExecute(config); + } + + public class ParamsTestRequiredProperty + { + private const string Expected = "a"; + + [Params(Expected)] + public required string ParamProperty { get; set; } + + [Benchmark] + public void Benchmark() => Assert.Equal(Expected, ParamProperty); + } +#endif } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/JitRuntimeValidationTest.cs b/tests/BenchmarkDotNet.IntegrationTests/JitRuntimeValidationTest.cs index 9a4910034c..df4427bb07 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/JitRuntimeValidationTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/JitRuntimeValidationTest.cs @@ -25,7 +25,7 @@ public JitRuntimeValidationTest(ITestOutputHelper output) : base(output) { } [InlineData(Jit.LegacyJit, Platform.X64, null)] [InlineData(Jit.RyuJit, Platform.X86, RyuJitNotAvailable)] [InlineData(Jit.RyuJit, Platform.X64, null)] - public void CheckClrOnWindows(Jit jit, Platform platform, string errorMessage) + public void CheckClrOnWindows(Jit jit, Platform platform, string? errorMessage) { Verify(ClrRuntime.Net462, jit, platform, errorMessage); } @@ -51,10 +51,10 @@ public static IEnumerable CheckCore_Arguments() [MemberData(nameof(CheckCore_Arguments))] public void CheckCore(Jit jit, Platform platform, string errorMessage) { - Verify(CoreRuntime.Core70, jit, platform, errorMessage); + Verify(CoreRuntime.Core80, jit, platform, errorMessage); } - private void Verify(Runtime runtime, Jit jit, Platform platform, string errorMessage) + private void Verify(Runtime runtime, Jit jit, Platform platform, string? errorMessage) { var logger = new OutputLogger(Output); var config = ManualConfig.CreateEmpty() diff --git a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs index 89ea7b5ba2..087d146822 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs @@ -25,7 +25,7 @@ public void BenchmarkCanAllocateMoreThan2Gb() var summary = BenchmarkRunner .Run( ManualConfig.CreateEmpty() - .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core70).WithPlatform(Platform.X64).WithId("Core")) + .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core")) .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("Framework")) .AddColumnProvider(DefaultColumnProviders.Instance) .AddLogger(new OutputLogger(output))); @@ -47,7 +47,7 @@ public void BenchmarkCanAllocateMoreThan2Gb() .Any()); Assert.Contains(".NET Framework", summary.AllRuntimes); - Assert.Contains(".NET 7.0", summary.AllRuntimes); + Assert.Contains(".NET 8.0", summary.AllRuntimes); } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index c1f2c622c6..7221a80595 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -8,6 +8,7 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.IntegrationTests.Xunit; @@ -34,11 +35,12 @@ public class MemoryDiagnoserTests public static IEnumerable GetToolchains() { - if (RuntimeInformation.IsOldMono) // https://github.com/mono/mono/issues/8397 - yield break; - yield return new object[] { Job.Default.GetToolchain() }; - yield return new object[] { InProcessEmitToolchain.Instance }; + // InProcessEmit reports flaky allocations in current .Net 8. + if (!RuntimeInformation.IsNetCore) + { + yield return new object[] { InProcessEmitToolchain.Instance }; + } } public class AccurateAllocations @@ -49,7 +51,7 @@ public class AccurateAllocations [Benchmark] public Task AllocateTask() => Task.FromResult(-12345); } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserIsAccurate(IToolchain toolchain) { @@ -69,22 +71,19 @@ public void MemoryDiagnoserIsAccurate(IToolchain toolchain) }); } - [FactEnvSpecific("We don't want to test NativeAOT twice (for .NET Framework 4.6.2 and .NET 7.0)", EnvRequirement.DotNetCoreOnly)] + [FactEnvSpecific("We don't want to test NativeAOT twice (for .NET Framework 4.6.2 and .NET 8.0)", EnvRequirement.DotNetCoreOnly)] public void MemoryDiagnoserSupportsNativeAOT() { - if (RuntimeInformation.IsMacOS()) + if (OsDetector.IsMacOS()) return; // currently not supported - MemoryDiagnoserIsAccurate( - NativeAotToolchain.CreateBuilder() - .UseNuGet("7.0.0", "https://api.nuget.org/v3/index.json") - .ToToolchain()); + MemoryDiagnoserIsAccurate(NativeAotToolchain.Net80); } - [FactEnvSpecific("We don't want to test MonoVM twice (for .NET Framework 4.6.2 and .NET 7.0)", EnvRequirement.DotNetCoreOnly)] + [FactEnvSpecific("We don't want to test MonoVM twice (for .NET Framework 4.6.2 and .NET 8.0)", EnvRequirement.DotNetCoreOnly)] public void MemoryDiagnoserSupportsModernMono() { - MemoryDiagnoserIsAccurate(MonoToolchain.Mono70); + MemoryDiagnoserIsAccurate(MonoToolchain.Mono80); } public class AllocatingGlobalSetupAndCleanup @@ -110,7 +109,7 @@ private void AllocateUntilGcWakesUp() } } - [Theory(Skip = "#1542 Tiered JIT Thread allocates memory in the background"), MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup(IToolchain toolchain) { @@ -120,24 +119,45 @@ public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup(IToolcha }); } - public class NoAllocationsAtAll + public class EmptyBenchmark { [Benchmark] public void EmptyMethod() { } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void EngineShouldNotInterfereAllocationResults(IToolchain toolchain) { - if (RuntimeInformation.IsFullFramework && toolchain.IsInProcess) + AssertAllocations(toolchain, typeof(EmptyBenchmark), new Dictionary + { + { nameof(EmptyBenchmark.EmptyMethod), 0 } + }); + } + + public class TimeConsumingBenchmark + { + [Benchmark] + public ulong TimeConsuming() { - return; // this test is flaky on Full Framework + var r = 1ul; + for (var i = 0; i < 50_000_000; i++) + { + r /= 1; + } + return r; } + } - AssertAllocations(toolchain, typeof(NoAllocationsAtAll), new Dictionary + // #1542 + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] + public void TieredJitShouldNotInterfereAllocationResults(IToolchain toolchain) + { + AssertAllocations(toolchain, typeof(TimeConsumingBenchmark), new Dictionary { - { nameof(NoAllocationsAtAll.EmptyMethod), 0 } - }); + { nameof(TimeConsumingBenchmark.TimeConsuming), 0 } + }, + disableTieredJit: false, iterationCount: 10); // 1 iteration is not enough to repro the problem } public class NoBoxing @@ -145,7 +165,7 @@ public class NoBoxing [Benchmark] public ValueTuple ReturnsValueType() => new ValueTuple(0); } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void EngineShouldNotIntroduceBoxing(IToolchain toolchain) { @@ -166,15 +186,10 @@ public class NonAllocatingAsynchronousBenchmarks [Benchmark] public ValueTask CompletedValueTaskOfT() => new ValueTask(default(int)); } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AwaitingTasksShouldNotInterfereAllocationResults(IToolchain toolchain) { - if (toolchain.IsInProcess) - { - return; // it's flaky: https://github.com/dotnet/BenchmarkDotNet/issues/1925 - } - AssertAllocations(toolchain, typeof(NonAllocatingAsynchronousBenchmarks), new Dictionary { { nameof(NonAllocatingAsynchronousBenchmarks.CompletedTask), 0 }, @@ -196,7 +211,7 @@ public void WithOperationsPerInvoke() private void DoNotInline(object left, object right) { } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AllocatedMemoryShouldBeScaledForOperationsPerInvoke(IToolchain toolchain) { @@ -222,67 +237,93 @@ public byte[] SixtyFourBytesArray() } } - [Theory(Skip = "#1542 Tiered JIT Thread allocates memory in the background"), MemberData(nameof(GetToolchains))] - //[TheoryNetCoreOnly("Only .NET Core 2.0+ API is bug free for this case"), MemberData(nameof(GetToolchains))] + [TheoryEnvSpecific("Full Framework cannot measure precisely enough for low invocation counts.", EnvRequirement.DotNetCoreOnly)] + [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) { long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word long arraySizeOverhead = IntPtr.Size; // array length + int warmupCount = OsDetector.IsMacOS() + ? 5 // Workaround setting for macos. https://github.com/dotnet/BenchmarkDotNet/issues/2779 + : 0; // Other OS don't need warmup AssertAllocations(toolchain, typeof(TimeConsuming), new Dictionary { { nameof(TimeConsuming.SixtyFourBytesArray), 64 + objectAllocationOverhead + arraySizeOverhead } - }); + }, warmupCount: warmupCount); } public class MultiThreadedAllocation { - public const int Size = 1_000_000; + public const int Size = 1024; public const int ThreadsCount = 10; + // We cache the threads in GlobalSetup and reuse them for each benchmark invocation + // to avoid measuring the cost of thread start and join, which varies across different runtimes. private Thread[] threads; + private volatile bool keepRunning = true; + private readonly Barrier barrier = new(ThreadsCount + 1); + private readonly CountdownEvent countdownEvent = new(ThreadsCount); - [IterationSetup] - public void SetupIteration() + [GlobalSetup] + public void Setup() { threads = Enumerable.Range(0, ThreadsCount) - .Select(_ => new Thread(() => GC.KeepAlive(new byte[Size]))) + .Select(_ => new Thread(() => + { + while (keepRunning) + { + barrier.SignalAndWait(); + GC.KeepAlive(new byte[Size]); + countdownEvent.Signal(); + } + })) .ToArray(); + foreach (var thread in threads) + { + thread.Start(); + } } - [Benchmark] - public void Allocate() + [GlobalCleanup] + public void Cleanup() { + keepRunning = false; + barrier.SignalAndWait(); foreach (var thread in threads) { - thread.Start(); thread.Join(); } } + + [Benchmark] + public void Allocate() + { + countdownEvent.Reset(ThreadsCount); + barrier.SignalAndWait(); + countdownEvent.Wait(); + } } - [TheoryEnvSpecific(".NET Core 3.0 preview6+ exposes a GC.GetTotalAllocatedBytes method which makes it possible to work", - EnvRequirement.DotNetCore30Only)] - [MemberData(nameof(GetToolchains))] + [TheoryEnvSpecific("Full Framework cannot measure precisely enough", EnvRequirement.DotNetCoreOnly)] + [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolchain) { long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word long arraySizeOverhead = IntPtr.Size; // array length long memoryAllocatedPerArray = (MultiThreadedAllocation.Size + objectAllocationOverhead + arraySizeOverhead); - long threadStartAndJoinOverhead = 112; // this is more or less a magic number taken from memory profiler - long allocatedMemoryPerThread = memoryAllocatedPerArray + threadStartAndJoinOverhead; AssertAllocations(toolchain, typeof(MultiThreadedAllocation), new Dictionary { - { nameof(MultiThreadedAllocation.Allocate), allocatedMemoryPerThread * MultiThreadedAllocation.ThreadsCount } + { nameof(MultiThreadedAllocation.Allocate), memoryAllocatedPerArray * MultiThreadedAllocation.ThreadsCount } }); } - private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary benchmarksAllocationsValidators) + private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary benchmarksAllocationsValidators, bool disableTieredJit = true, int iterationCount = 1, int warmupCount = 0) { - var config = CreateConfig(toolchain); + var config = CreateConfig(toolchain, disableTieredJit, iterationCount, warmupCount); var benchmarks = BenchmarkConverter.TypeToBenchmarks(benchmarkType, config); var summary = BenchmarkRunner.Run(benchmarks); @@ -318,20 +359,37 @@ private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Diction } } - private IConfig CreateConfig(IToolchain toolchain) - => ManualConfig.CreateEmpty() - .AddJob(Job.ShortRun - .WithEvaluateOverhead(false) // no need to run idle for this test - .WithWarmupCount(0) // don't run warmup to save some time for our CI runs - .WithIterationCount(1) // single iteration is enough for us - .WithGcForce(false) - .WithGcServer(false) - .WithGcConcurrent(false) - .WithEnvironmentVariable("COMPlus_TieredCompilation", "0") // Tiered JIT can allocate some memory on a background thread, let's disable it to make our tests less flaky (#1542) - .WithToolchain(toolchain)) + private IConfig CreateConfig(IToolchain toolchain, + // Tiered JIT can allocate some memory on a background thread, let's disable it by default to make our tests less flaky (#1542). + // This was mostly fixed in net7.0, but tiered jit thread is not guaranteed to not allocate, so we disable it just in case. + bool disableTieredJit = true, + // Single iteration is enough for most of the tests. + int iterationCount = 1, + // Don't run warmup by default to save some time for our CI runs + int warmupCount = 0) + { + var job = Job.ShortRun + .WithEvaluateOverhead(false) // no need to run idle for this test + .WithWarmupCount(warmupCount) + .WithIterationCount(iterationCount) + .WithGcForce(false) + .WithGcServer(false) + .WithGcConcurrent(false) + // To prevent finalizers allocating out of our control, we hang the finalizer thread. + // https://github.com/dotnet/runtime/issues/101536#issuecomment-2077647417 + .WithEnvironmentVariable(Engines.Engine.UnitTestBlockFinalizerEnvKey, Engines.Engine.UnitTestBlockFinalizerEnvValue) + .WithToolchain(toolchain); + return ManualConfig.CreateEmpty() + .AddJob(disableTieredJit + ? job.WithEnvironmentVariables( + new EnvironmentVariable("DOTNET_TieredCompilation", "0"), + new EnvironmentVariable("COMPlus_TieredCompilation", "0") + ) + : job) .AddColumnProvider(DefaultColumnProviders.Instance) .AddDiagnoser(MemoryDiagnoser.Default) .AddLogger(toolchain.IsInProcess ? ConsoleLogger.Default : new OutputLogger(output)); // we can't use OutputLogger for the InProcess toolchains because it allocates memory on the same thread + } // note: don't copy, never use in production systems (it should work but I am not 100% sure) private int CalculateRequiredSpace() diff --git a/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs index 4313dfbec1..17309832ef 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs @@ -4,16 +4,23 @@ using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; +using Xunit.Abstractions; namespace BenchmarkDotNet.IntegrationTests { public class MonoTests : BenchmarkTestExecutor { + public MonoTests(ITestOutputHelper output) : base(output) { } + [FactEnvSpecific("UseMonoRuntime option is available in .NET Core only starting from .NET 6", EnvRequirement.DotNetCoreOnly)] - public void Mono70IsSupported() + public void Mono80IsSupported() { - var config = ManualConfig.CreateEmpty().AddJob(Job.Dry.WithRuntime(MonoRuntime.Mono70)); + var logger = new OutputLogger(Output); + var config = ManualConfig.CreateEmpty() + .AddLogger(logger) + .AddJob(Job.Dry.WithRuntime(MonoRuntime.Mono80)); CanExecute(config); } @@ -27,11 +34,11 @@ public void Check() throw new Exception("This is not Mono runtime"); } - if (RuntimeInformation.GetCurrentRuntime() != MonoRuntime.Mono70) + if (RuntimeInformation.GetCurrentRuntime() != MonoRuntime.Mono80) { throw new Exception("Incorrect runtime detection"); } } } } -} +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs b/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs index 938a1e2011..50a929bd83 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs @@ -29,7 +29,7 @@ public void SingleBenchmarkCanBeExecutedForMultipleRuntimes() var summary = BenchmarkRunner .Run( ManualConfig.CreateEmpty() - .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core70).WithPlatform(Platform.X64).WithId("Core")) + .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core")) .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithId("Framework")) .AddColumnProvider(DefaultColumnProviders.Instance) .AddLogger(new OutputLogger(output))); @@ -51,7 +51,7 @@ public void SingleBenchmarkCanBeExecutedForMultipleRuntimes() .Any()); Assert.Contains(".NET Framework", summary.AllRuntimes); - Assert.Contains(".NET 7.0", summary.AllRuntimes); + Assert.Contains(".NET 8.0", summary.AllRuntimes); } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs index ce39b0572e..169fb3a0b3 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs @@ -1,6 +1,7 @@ using System; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Jobs; @@ -22,7 +23,7 @@ public void LatestNativeAotVersionIsSupported() return; if (ContinuousIntegration.IsGitHubActionsOnWindows()) // no native dependencies installed return; - if (RuntimeInformation.IsMacOS()) + if (OsDetector.IsMacOS()) return; // currently not supported var toolchain = NativeAotToolchain.CreateBuilder().UseNuGet().IlcInstructionSet(IsAvx2Supported() ? "avx2" : "").ToToolchain(); diff --git a/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs index 182acc5309..e9f9469f13 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs @@ -1,4 +1,7 @@ using System; +using System.Collections.Immutable; +using System.Linq; +using System.Reflection; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -6,7 +9,6 @@ using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.Roslyn; -using Newtonsoft.Json; using Xunit; using Xunit.Abstractions; @@ -18,25 +20,33 @@ public NuGetReferenceTests(ITestOutputHelper output) : base(output) { } - [FactEnvSpecific("For some reason this test is unstable on Ubuntu for both AzureDevOps and Travis CI", EnvRequirement.NonLinux)] + [Fact] + [Obsolete] public void UserCanSpecifyCustomNuGetPackageDependency() { var toolchain = RuntimeInformation.GetCurrentRuntime().GetToolchain(preferMsBuildToolchains: true); - var job = Job.Dry.WithToolchain(toolchain).WithNuGet("Newtonsoft.Json", "13.0.2"); + const string targetVersion = "9.0.5"; + + var job = Job.Dry.WithToolchain(toolchain).WithNuGet("System.Collections.Immutable", targetVersion); var config = CreateSimpleConfig(job: job); - CanExecute(config); + var report = CanExecute(config); + + // Validate NuGet package version output message + var stdout = GetSingleStandardOutput(report); + Assert.Contains($"System.Collections.Immutable: {targetVersion}", stdout); } [FactEnvSpecific("Roslyn toolchain does not support .NET Core", EnvRequirement.FullFrameworkOnly)] + [Obsolete] public void RoslynToolchainDoesNotSupportNuGetPackageDependency() { var toolchain = RoslynToolchain.Instance; - var unsupportedJob = Job.Dry.WithToolchain(toolchain).WithNuGet("Newtonsoft.Json", "13.0.2"); + var unsupportedJob = Job.Dry.WithToolchain(toolchain).WithNuGet("System.Collections.Immutable", "9.0.5"); var unsupportedJobConfig = CreateSimpleConfig(job: unsupportedJob); - var unsupportedJobBenchmark = BenchmarkConverter.TypeToBenchmarks(typeof(WithCallToNewtonsoft), unsupportedJobConfig); + var unsupportedJobBenchmark = BenchmarkConverter.TypeToBenchmarks(typeof(WithCallToImmutableArray), unsupportedJobConfig); foreach (var benchmarkCase in unsupportedJobBenchmark.BenchmarksCases) { @@ -45,16 +55,39 @@ public void RoslynToolchainDoesNotSupportNuGetPackageDependency() var supportedJob = Job.Dry.WithToolchain(toolchain); var supportedConfig = CreateSimpleConfig(job: supportedJob); - var supportedBenchmark = BenchmarkConverter.TypeToBenchmarks(typeof(WithCallToNewtonsoft), supportedConfig); + var supportedBenchmark = BenchmarkConverter.TypeToBenchmarks(typeof(WithCallToImmutableArray), supportedConfig); foreach (var benchmarkCase in supportedBenchmark.BenchmarksCases) { Assert.Empty(toolchain.Validate(benchmarkCase, BenchmarkRunnerClean.DefaultResolver)); } } - public class WithCallToNewtonsoft + public class WithCallToImmutableArray { - [Benchmark] public void SerializeAnonymousObject() => JsonConvert.SerializeObject(new { hello = "world", price = 1.99, now = DateTime.UtcNow }); + private readonly double[] values; + + public WithCallToImmutableArray() + { + var rand = new Random(Seed: 0); + values = Enumerable.Range(1, 10_000) + .Select(x => rand.NextDouble()) + .ToArray(); + + // Gets assembly version text from AssemblyInformationalVersion attribute. + var version = typeof(ImmutableArray).Assembly + .GetCustomAttribute()! + .InformationalVersion + .Split('+')[0]; + + // Print referenced NuGet package version to stdout. + Console.WriteLine($"System.Collections.Immutable: {version}"); + } + + [Benchmark] + public void ToImmutableArrayBenchmark() + { + var results = values.ToImmutableArray(); + } } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/ParamSourceTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ParamSourceTests.cs index ed105f8290..82255131fd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ParamSourceTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ParamSourceTests.cs @@ -14,7 +14,7 @@ namespace BenchmarkDotNet.IntegrationTests { - public class ParamSourceTests: BenchmarkTestExecutor + public class ParamSourceTests : BenchmarkTestExecutor { public ParamSourceTests(ITestOutputHelper output) : base(output) { } @@ -96,7 +96,7 @@ public static IEnumerable GetSource() public int Benchmark() => ParamsTarget?.Data ?? 0; } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void PrivateClassWithPublicInterface_Succeeds(IToolchain toolchain) => CanExecuteWithExtraInfo(typeof(PrivateClassWithPublicInterface), toolchain); public class PrivateClassWithPublicInterface_Array @@ -116,7 +116,7 @@ public IEnumerable GetSource() public int Benchmark() => ParamsTarget?.Sum(p => p?.Data ?? 0) ?? 0; } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void PrivateClassWithPublicInterface_Array_Succeeds(IToolchain toolchain) => CanExecuteWithExtraInfo(typeof(PrivateClassWithPublicInterface_Array), toolchain); public class PrivateClassWithPublicInterface_Enumerable @@ -137,7 +137,7 @@ public IEnumerable> GetSource() public int Benchmark() => ParamsTarget?.Sum(p => p?.Data ?? 0) ?? 0; } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void PrivateClassWithPublicInterface_Enumerable_Succeeds(IToolchain toolchain) => CanExecuteWithExtraInfo(typeof(PrivateClassWithPublicInterface_Enumerable), toolchain); public class PrivateClassWithPublicInterface_AsObject @@ -156,7 +156,7 @@ public static IEnumerable GetSource() public int Benchmark() => ParamsTarget?.Data ?? 0; } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void PrivateClassWithPublicInterface_AsObject_Succeeds(IToolchain toolchain) => CanExecuteWithExtraInfo(typeof(PrivateClassWithPublicInterface_AsObject), toolchain); public class PublicSource @@ -204,5 +204,43 @@ public void SourceWithExplicitCastToTarget_InProcessToolchain_Throws() // public void SourceWithExplicitCastToTarget_Succeeds(IToolchain toolchain) => CanExecuteWithExtraInfo(typeof(SourceWithExplicitCastToTarget), toolchain); Assert.ThrowsAny(() => CanExecuteWithExtraInfo(typeof(SourceWithExplicitCastToTarget), InProcessEmitToolchain.Instance)); } + + public abstract class OverridePropertyBase + { + public abstract int[] GetSourceProperty { get; } + + [ParamsSource(nameof(GetSourceProperty))] + public int ParamsTarget { get; set; } + } + + public class OverrideProperty : OverridePropertyBase + { + public override int[] GetSourceProperty => new int[] { 1, 2, 3 }; + + [Benchmark] + public int Benchmark() => ParamsTarget; + } + + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + public void OverrideProperty_Succeeds(IToolchain toolchain) => CanExecuteWithExtraInfo(typeof(OverrideProperty), toolchain); + + public abstract class OverrideMethodBase + { + public abstract int[] GetSourceMethod(); + + [ParamsSource(nameof(GetSourceMethod))] + public int ParamsTarget { get; set; } + } + + public class OverrideMethod : OverrideMethodBase + { + public override int[] GetSourceMethod() => new int[] { 1, 2, 3 }; + + [Benchmark] + public int Benchmark() => ParamsTarget; + } + + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + public void OverrideMethod_Succeeds(IToolchain toolchain) => CanExecuteWithExtraInfo(typeof(OverrideMethod), toolchain); } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs index 0285856bcb..0f703bf1cd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs @@ -217,5 +217,28 @@ public void Test() throw new ArgumentException($"{nameof(StaticParamProperty)} has wrong value: {StaticParamProperty}!"); } } + +#if NET8_0_OR_GREATER + [Fact] + public void ParamsSupportRequiredProperty() + { + var summary = CanExecute(); + var standardOutput = GetCombinedStandardOutput(summary); + + foreach (var param in new[] { "a", "b" }) + { + Assert.Contains($"// ### New Parameter {param} ###", standardOutput); + } + } + + public class ParamsTestRequiredProperty + { + [Params("a", "b")] + public required string ParamProperty { get; set; } + + [Benchmark] + public void Benchmark() => Console.WriteLine($"// ### New Parameter {ParamProperty} ###"); + } +#endif } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/PowerRequest.cs b/tests/BenchmarkDotNet.IntegrationTests/PowerRequest.cs new file mode 100644 index 0000000000..676ad23d40 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/PowerRequest.cs @@ -0,0 +1,9 @@ +namespace BenchmarkDotNet.IntegrationTests; + +internal class PowerRequest(string requestType, string requesterType, string requesterName, string? reason) +{ + public string RequestType { get; } = requestType; + public string RequesterType { get; } = requesterType; + public string RequesterName { get; } = requesterName; + public string? Reason { get; } = reason; +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/PowerRequestsParser.cs b/tests/BenchmarkDotNet.IntegrationTests/PowerRequestsParser.cs new file mode 100644 index 0000000000..bb634b5228 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/PowerRequestsParser.cs @@ -0,0 +1,187 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace BenchmarkDotNet.IntegrationTests; + +/// +/// Parses the output of 'powercfg /requests' command into a list of s. +/// +/// +/// +/// Not using Sprache. It is superseded by Superpower. +/// Not using Superpower. I gained more knowledge +/// implementing this class from scratch. +/// +/// Example input: +/// +/// DISPLAY: +/// [PROCESS] \Device\HarddiskVolume3\Program Files (x86)\Google\Chrome\Application\chrome.exe +/// Video Wake Lock +/// +/// SYSTEM: +/// [DRIVER] Realtek High Definition Audio(SST) ... +/// Er wordt momenteel een audiostream gebruikt. +/// [PROCESS] \Device\HarddiskVolume3\...\NoSleep.exe +/// [PROCESS] \Device\HarddiskVolume3\Program Files (x86)\Google\Chrome\Application\chrome.exe +/// Video Wake Lock +/// +/// AWAYMODE: +/// None. +/// +/// EXECUTION: +/// [PROCESS] \Device\HarddiskVolume3\Program Files (x86)\Google\Chrome\Application\chrome.exe +/// Playing audio +/// +/// PERFBOOST: +/// None. +/// +/// ACTIVELOCKSCREEN: +/// None. +/// +/// +/// +internal class PowerRequestsParser +{ + /// + /// Parses output of 'powercfg /requests' into a list of s. + /// + /// + /// + /// This method takes a list of s. Examines next token and decides how to + /// parse. + /// + /// + /// Output of 'powercfg /requests'. + public static IEnumerable Parse(string input) + { + using TokenStream tokens = new TokenStream(Tokens(input)); + while (tokens.TryPeek().HasValue) + { + foreach (PowerRequest item in ParseRequestType(tokens)) + { + yield return item; + } + } + } + + private static IEnumerable ParseRequestType(TokenStream tokens) + { + Token requestType = tokens.Take(TokenType.RequestType); + if (tokens.Peek().TokenType == TokenType.RequesterType) + { + while (tokens.Peek().TokenType == TokenType.RequesterType) + { + yield return ParseRequesterType(requestType, tokens); + } + } + else + { + _ = tokens.Take(TokenType.None); + } + _ = tokens.Take(TokenType.EmptyLine); + } + + private static PowerRequest ParseRequesterType(Token requestType, TokenStream tokens) + { + Token requesterType = tokens.Take(TokenType.RequesterType); + Token requesterName = tokens.Take(TokenType.RequesterName); + Token? reason = null; + if (tokens.Peek().TokenType == TokenType.Reason) + { + reason = tokens.Take(TokenType.Reason); + } + return new PowerRequest(requestType.Value, requesterType.Value, requesterName.Value, reason?.Value); + } + + /// + /// Converts the input into a list of s. + /// + /// + /// + /// Looking at above sample, tokenizing is made simple when done line by line. Each line + /// contains one or two s. + /// + /// + /// Output of 'powercfg /requests'. + private static IEnumerable Tokens(string input) + { + // Contrary to calling input.Split('\r', '\n'), StringReader's ReadLine method does not + // return an empty string when CR is followed by LF. + StringReader reader = new StringReader(input); + string? line; + while ((line = reader.ReadLine()) != null) + { + if (line.Length == 0) + { + yield return new Token(TokenType.EmptyLine, ""); + } + else if (line[line.Length - 1] == ':') + { + yield return new Token(TokenType.RequestType, line.Substring(0, line.Length - 1).ToString()); + } + else if (string.Equals(line, "None.", StringComparison.InvariantCulture)) + { + yield return new Token(TokenType.None, line); + } + else if (line[0] == '[') + { + int pos = line.IndexOf(']'); + yield return new Token(TokenType.RequesterType, line.Substring(1, pos - 1)); + yield return new Token(TokenType.RequesterName, line.Substring(pos + 2)); + } + else + { + yield return new Token(TokenType.Reason, line); + } + } + } + + /// + /// Adds and to an of + /// s. + /// + /// + private class TokenStream(IEnumerable tokens) : IDisposable + { + private readonly IEnumerator tokens = tokens.GetEnumerator(); + private Token? cached; + + public Token? TryPeek() => cached ??= tokens.MoveNext() ? tokens.Current : null; + + public Token Peek() => TryPeek() ?? throw new EndOfStreamException(); + + public Token Take(TokenType requestType) + { + Token peek = Peek(); + if (peek.TokenType == requestType) + { + cached = null; + return peek; + } + else + { + throw new InvalidCastException($"Unexpected Token of type '{peek.TokenType}'. Expected type '{requestType}'."); + } + } + + public void Dispose() => tokens.Dispose(); + } + + private enum TokenType + { + EmptyLine, + None, + Reason, + RequesterName, + RequesterType, + RequestType + } + + private readonly struct Token(TokenType tokenType, string value) + { + public TokenType TokenType { get; } = tokenType; + + public string Value { get; } = value; + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/TailCallDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/TailCallDiagnoserTests.cs index ab9fbbb645..71988ba7c9 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/TailCallDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/TailCallDiagnoserTests.cs @@ -53,7 +53,7 @@ public class NonTailCallBenchmarks } [TheoryEnvSpecific(WindowsOnly, EnvRequirement.WindowsOnly)] - [MemberData(nameof(GetJits))] + [MemberData(nameof(GetJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void TailCallDiagnoserCatchesTailCallEvents(Jit jit, Platform platform, Runtime runtime) { @@ -63,7 +63,7 @@ public void TailCallDiagnoserCatchesTailCallEvents(Jit jit, Platform platform, R } [TheoryEnvSpecific(WindowsOnly, EnvRequirement.WindowsOnly)] - [MemberData(nameof(GetJits))] + [MemberData(nameof(GetJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void TailCallDiagnoserNotCatchesTailCallEvents(Jit jit, Platform platform, Runtime runtime) { diff --git a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs index 220991796d..c1a45d4e01 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs @@ -14,6 +14,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Portability; using Xunit; @@ -32,17 +33,15 @@ public static IEnumerable GetToolchains() yield return new object[] { Job.Default.GetToolchain() }; if (!ContinuousIntegration.IsGitHubActionsOnWindows() // no native dependencies - && !RuntimeInformation.IsMacOS()) // currently not supported + && !OsDetector.IsMacOS()) // currently not supported { - yield return new object[]{ NativeAotToolchain.CreateBuilder() - .UseNuGet("7.0.0", "https://api.nuget.org/v3/index.json") - .ToToolchain() }; + yield return new object[]{ NativeAotToolchain.Net80 }; } // TODO: Support InProcessEmitToolchain.Instance // yield return new object[] { InProcessEmitToolchain.Instance }; } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void CompletedWorkItemCountIsAccurate(IToolchain toolchain) { var config = CreateConfig(toolchain); @@ -83,7 +82,7 @@ public void CompleteOneWorkItem() public void DoNothing() { } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void LockContentionCountIsAccurate(IToolchain toolchain) { var config = CreateConfig(toolchain); diff --git a/tests/BenchmarkDotNet.IntegrationTests/ValuesReturnedByBenchmarkTest.cs b/tests/BenchmarkDotNet.IntegrationTests/ValuesReturnedByBenchmarkTest.cs index f6a5d10428..e4eb071029 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ValuesReturnedByBenchmarkTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ValuesReturnedByBenchmarkTest.cs @@ -22,7 +22,7 @@ public static IEnumerable GetToolchains() => new[] new object[] { InProcessEmitToolchain.Instance }, }; - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void AnyValueCanBeReturned(IToolchain toolchain) => CanExecute(ManualConfig.CreateEmpty().AddJob(Job.Dry.WithToolchain(toolchain))); public class ValuesReturnedByBenchmark diff --git a/tests/BenchmarkDotNet.IntegrationTests/WakeLockTests.cs b/tests/BenchmarkDotNet.IntegrationTests/WakeLockTests.cs new file mode 100644 index 0000000000..cf8d0d39b9 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/WakeLockTests.cs @@ -0,0 +1,179 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Tests.XUnit; +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.Versioning; +using System.Threading; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.IntegrationTests; + +public class WakeLockTests : BenchmarkTestExecutor +{ + private const string PingEventName = @"Global\WakeLockTests-ping"; + private const string PongEventName = @"Global\WakeLockTests-pong"; + private static readonly TimeSpan testTimeout = TimeSpan.FromMinutes(1); + private readonly OutputLogger logger; + + public WakeLockTests(ITestOutputHelper output) : base(output) + { + logger = new OutputLogger(Output); + } + + [Fact] + public void ConfigurationDefaultValue() + { + Assert.Equal(WakeLockType.System, DefaultConfig.Instance.WakeLock); + Assert.Equal(WakeLockType.None, new DebugBuildConfig().WakeLock); + Assert.Equal(WakeLockType.None, new DebugInProcessConfig().WakeLock); + } + + [TheoryEnvSpecific(EnvRequirement.NonWindows)] + [InlineData(WakeLockType.None)] + [InlineData(WakeLockType.System)] + [InlineData(WakeLockType.Display)] + public void WakeLockIsWindowsOnly(WakeLockType wakeLockType) + { + using IDisposable wakeLock = WakeLock.Request(wakeLockType, "dummy", logger); + Assert.Null(wakeLock); + } + + [FactEnvSpecific(EnvRequirement.WindowsOnly)] + public void WakeLockSleepOrDisplayIsAllowed() + { + using IDisposable wakeLock = WakeLock.Request(WakeLockType.None, "dummy", logger); + Assert.Null(wakeLock); + } + + [FactEnvSpecific(EnvRequirement.WindowsOnly, EnvRequirement.NeedsPrivilegedProcess)] + public void WakeLockRequireSystem() + { + using (IDisposable wakeLock = WakeLock.Request(WakeLockType.System, "WakeLockTests", logger)) + { + Assert.NotNull(wakeLock); + Assert.Equal("SYSTEM", GetPowerRequests("WakeLockTests")); + } + Assert.Equal("", GetPowerRequests()); + } + + [FactEnvSpecific(EnvRequirement.WindowsOnly, EnvRequirement.NeedsPrivilegedProcess)] + public void WakeLockRequireDisplay() + { + using (IDisposable wakeLock = WakeLock.Request(WakeLockType.Display, "WakeLockTests", logger)) + { + Assert.NotNull(wakeLock); + Assert.Equal("DISPLAY, SYSTEM", GetPowerRequests("WakeLockTests")); + } + Assert.Equal("", GetPowerRequests()); + } + + [FactEnvSpecific(EnvRequirement.NonWindows)] + public void BenchmarkRunnerIgnoresWakeLock() => + _ = CanExecute(fullValidation: false); + + [WakeLock(WakeLockType.Display)] + public class IgnoreWakeLock + { + [Benchmark] public void Sleep() { } + } + +#if !NET462 + [SupportedOSPlatform("windows")] +#endif + [TheoryEnvSpecific(EnvRequirement.WindowsOnly, EnvRequirement.NeedsPrivilegedProcess)] + [InlineData(typeof(Default), "SYSTEM")] + [InlineData(typeof(None), "")] + [InlineData(typeof(RequireSystem), "SYSTEM")] + [InlineData(typeof(RequireDisplay), "DISPLAY, SYSTEM")] + public async Task BenchmarkRunnerAcquiresWakeLock(Type type, string expected) + { + using EventWaitHandle + ping = new EventWaitHandle(false, EventResetMode.AutoReset, PingEventName), + pong = new EventWaitHandle(false, EventResetMode.AutoReset, PongEventName); + string pwrRequests = null; + Task task = WaitForBenchmarkRunningAndGetPowerRequests(); + _ = CanExecute(type, fullValidation: false); + await task; + + Assert.Equal(expected, pwrRequests); + + async Task WaitForBenchmarkRunningAndGetPowerRequests() + { + await AsTask(ping, testTimeout); + pwrRequests = GetPowerRequests("BenchmarkDotNet Running Benchmarks"); + pong.Set(); + } + } + + public class Default : Base { } + + [WakeLock(WakeLockType.None)] public class None : Base { } + + [WakeLock(WakeLockType.System)] public class RequireSystem : Base { } + + [WakeLock(WakeLockType.Display)] public class RequireDisplay : Base { } + + public class Base + { + [Benchmark] +#if !NET462 + [SupportedOSPlatform("windows")] +#endif + public void SignalBenchmarkRunningAndWaitForGetPowerRequests() + { + using EventWaitHandle + ping = EventWaitHandle.OpenExisting(PingEventName), + pong = EventWaitHandle.OpenExisting(PongEventName); + ping.Set(); + pong.WaitOne(testTimeout); + } + } + + private string GetPowerRequests(string? expectedReason = null) + { + string pwrRequests = ProcessHelper.RunAndReadOutput("powercfg", "/requests"); + Output.WriteLine(pwrRequests); // Useful to analyse failing tests. + string fileName = Process.GetCurrentProcess().MainModule.FileName; + string mustEndWith = fileName.Substring(Path.GetPathRoot(fileName).Length); + + return string.Join(", ", + from pr in PowerRequestsParser.Parse(pwrRequests) + where + pr.RequesterName.EndsWith(mustEndWith, StringComparison.InvariantCulture) && + string.Equals(pr.RequesterType, "PROCESS", StringComparison.InvariantCulture) && + (expectedReason == null || string.Equals(pr.Reason, expectedReason, StringComparison.InvariantCulture)) + select pr.RequestType); + } + + private Task AsTask(WaitHandle waitHandle, TimeSpan timeout) + { + TaskCompletionSource tcs = new TaskCompletionSource(); + RegisteredWaitHandle rwh = null; + rwh = ThreadPool.RegisterWaitForSingleObject( + waitHandle, + (object state, bool timedOut) => + { + rwh.Unregister(null); + if (timedOut) + { + tcs.SetException(new TimeoutException()); + } + else + { + tcs.SetResult(true); + } + }, + null, + timeout, + true); + return tcs.Task; + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs new file mode 100644 index 0000000000..6d3519a9c5 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs @@ -0,0 +1,57 @@ +using System; +using System.IO; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Tests.XUnit; +using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Toolchains.MonoWasm; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.IntegrationTests +{ + /// + /// In order to run WasmTests locally, the following prerequisites are required: + /// * Install wasm-tools workload: `BenchmarkDotNet/build.cmd install-wasm-tools` + /// * Install npm + /// * Install v8: `npm install jsvu -g && jsvu --os=default --engines=v8` + /// * Add `$HOME/.jsvu/bin` to PATH + /// * Run tests using .NET SDK from `BenchmarkDotNet/.dotnet/` + /// + public class WasmTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) + { + [FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)] + public void WasmIsSupported() + { + var dotnetVersion = "net8.0"; + var logger = new OutputLogger(Output); + var netCoreAppSettings = new NetCoreAppSettings(dotnetVersion, null, "Wasm"); + var mainJsPath = Path.Combine(AppContext.BaseDirectory, "AppBundle", "test-main.js"); + + var config = ManualConfig.CreateEmpty() + .AddLogger(logger) + .AddJob(Job.Dry + .WithArguments([new MsBuildArgument($"/p:WasmMainJSPath={mainJsPath}")]) + .WithRuntime(new WasmRuntime(dotnetVersion, moniker: RuntimeMoniker.WasmNet80, javaScriptEngineArguments: "--expose_wasm --module")) + .WithToolchain(WasmToolchain.From(netCoreAppSettings))) + .WithOption(ConfigOptions.GenerateMSBuildBinLog, true); + + CanExecute(config); + } + + public class WasmBenchmark + { + [Benchmark] + public void Check() + { + if (!RuntimeInformation.IsWasm) + { + throw new Exception("Incorrect runtime detection"); + } + } + } + } +} diff --git a/tests/BenchmarkDotNet.Tests/Analysers/OutliersAnalyserTests.cs b/tests/BenchmarkDotNet.Tests/Analysers/OutliersAnalyserTests.cs index c1368adf94..e8f42d2366 100644 --- a/tests/BenchmarkDotNet.Tests/Analysers/OutliersAnalyserTests.cs +++ b/tests/BenchmarkDotNet.Tests/Analysers/OutliersAnalyserTests.cs @@ -10,12 +10,8 @@ namespace BenchmarkDotNet.Tests.Analysers { - public class OutliersAnalyserTests + public class OutliersAnalyserTests(ITestOutputHelper output) { - private readonly ITestOutputHelper output; - - public OutliersAnalyserTests(ITestOutputHelper output) => this.output = output; - [Theory] [InlineData(0, 0, "")] [InlineData(1, 1, "1 outlier was removed")] @@ -64,9 +60,9 @@ public void RangeMessageTest(int lowerOutliers, int upperOutliers, string expect var cultureInfo = TestCultureInfo.Instance; string actualMessage = OutliersAnalyser.GetMessage(s.AllOutliers, s.AllOutliers, s.LowerOutliers, s.UpperOutliers, cultureInfo).ToAscii(); output.WriteLine("Values : " + - string.Join(", ", values.Take(5).Select(x => TimeInterval.FromNanoseconds(x).ToString(cultureInfo, "N2"))) + + string.Join(", ", values.Take(5).Select(x => TimeInterval.FromNanoseconds(x).ToDefaultString("N2"))) + ", ..., " + - string.Join(", ", values.Skip(values.Count - 5).Select(x => TimeInterval.FromNanoseconds(x).ToString(cultureInfo, "N2")))); + string.Join(", ", values.Skip(values.Count - 5).Select(x => TimeInterval.FromNanoseconds(x).ToDefaultString("N2")))); output.WriteLine("Actual : " + actualMessage); output.WriteLine("Expected : " + expectedMessage); Assert.Equal(expectedMessage, actualMessage); diff --git a/tests/BenchmarkDotNet.Tests/Analysers/ZeroMeasurementHelperTests.cs b/tests/BenchmarkDotNet.Tests/Analysers/ZeroMeasurementHelperTests.cs index d0bcb15cae..9a6679e3ec 100644 --- a/tests/BenchmarkDotNet.Tests/Analysers/ZeroMeasurementHelperTests.cs +++ b/tests/BenchmarkDotNet.Tests/Analysers/ZeroMeasurementHelperTests.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using BenchmarkDotNet.Analysers; +using Perfolizer; using Xunit; namespace BenchmarkDotNet.Tests.Analysers @@ -9,7 +10,7 @@ public class ZeroMeasurementHelperTests #region OneSampleTests // let processor frequency ~ 3.30 GHz - private static double ThresholdMock = 0.2702d / 2; + private const double ThresholdMock = 0.2702d / 2; /* Test distributions inspired by data from method @@ -30,16 +31,14 @@ and RyuJit x64 [InlineData(0.27049, 0.27410, 0.27433, 0.27468, 0.27476, 0.27591, 0.27644, 0.27704, 0.27724, 0.27766, 0.27792, 0.27878, 0.28025)] public void OneSample_Around_One_CPU_Cycle_Method(params double[] distribution) { - var zeroMeasurement = ZeroMeasurementHelper.CheckZeroMeasurementOneSample(distribution, ThresholdMock); - Assert.False(zeroMeasurement); + Assert.True(ZeroMeasurementHelper.IsNoticeable(new Sample(distribution), ThresholdMock)); } [Theory] [InlineData(0.2703, 0.2701, 0.2703, 0.2701, 0.2703, 0.2701, 0.2703, 0.2701, 0.2703, 0.2701, 0.2703, 0.2701)] public void OneSample_Exactly_One_CPU_Cycle_Method(params double[] distribution) { - var zeroMeasurement = ZeroMeasurementHelper.CheckZeroMeasurementOneSample(distribution, ThresholdMock); - Assert.False(zeroMeasurement); + Assert.True(ZeroMeasurementHelper.IsNoticeable(new Sample(distribution), ThresholdMock)); } /* @@ -57,11 +56,11 @@ and RyuJit x64 [Theory] [InlineData(0d, 0d, 0.00191d, 0.00530d, 0.00820d, 0.01383d, 0.01617d, 0.02183d, 0.02421d, 0.03640d, 0.03726d, 0.04894d, 0.05122d, 0.05924d, 0.06183d)] [InlineData(0d, 0d, 0d, 0d, 0d, 0d, 0d, 0.00138d, 0.00482d, 0.00616d, 0.01318d, 0.02266d, 0.03048d, 0.03144d)] - [InlineData(0.02203d, 0.02523d, 0.02567d, 0.02706d, 0.03048d, 0.03461d, 0.03953d, 0.04127d, 0.04396d, 0.04939d, 0.05361d, 0.05670d, 0.06394d, 0.06812d, 0.06901d)] + [InlineData(0.02203d, 0.02523d, 0.02567d, 0.02706d, 0.03048d, 0.03461d, 0.03953d, 0.04127d, 0.04396d, 0.04939d, 0.05361d, 0.05670d, 0.06394d, 0.06812d, + 0.06901d)] public void OneSample_Less_Than_One_CPU_Cycle_Method(params double[] distribution) { - var zeroMeasurement = ZeroMeasurementHelper.CheckZeroMeasurementOneSample(distribution, ThresholdMock); - Assert.True(zeroMeasurement); + Assert.True(ZeroMeasurementHelper.IsNegligible(new Sample(distribution), ThresholdMock)); } /* @@ -72,8 +71,7 @@ Sometimes appears distributions with all zero values [InlineData(0d, 0d, 0d, 0d, 0d, 0d, 0d, 0d, 0d, 0d, 0d, 0d, 0d)] public void OneSample_Exactly_Zero_ns_Method(params double[] distribution) { - var zeroMeasurement = ZeroMeasurementHelper.CheckZeroMeasurementOneSample(distribution, ThresholdMock); - Assert.True(zeroMeasurement); + Assert.True(ZeroMeasurementHelper.IsNegligible(new Sample(distribution), ThresholdMock)); } /* @@ -85,18 +83,20 @@ Test distributions inspired by data from method and RyuJit x64 */ [Theory] - [InlineData(0.57079d, 0.57469d, 0.57990d, 0.58025d, 0.58532d, 0.59250d, 0.59442d, 0.59487d, 0.59522d, 0.59619d, 0.59756d, 0.59802d, 0.60120d, 0.60813d, 0.61592d)] - [InlineData(0.57106d, 0.57168d, 0.57326d, 0.57587d, 0.57958d, 0.58982d, 0.59493d, 0.59950d, 0.61413d, 0.62000d, 0.62607d, 0.63209d, 0.63730d, 0.65048d, 0.65119d)] + [InlineData(0.57079d, 0.57469d, 0.57990d, 0.58025d, 0.58532d, 0.59250d, 0.59442d, 0.59487d, 0.59522d, 0.59619d, 0.59756d, 0.59802d, 0.60120d, 0.60813d, + 0.61592d)] + [InlineData(0.57106d, 0.57168d, 0.57326d, 0.57587d, 0.57958d, 0.58982d, 0.59493d, 0.59950d, 0.61413d, 0.62000d, 0.62607d, 0.63209d, 0.63730d, 0.65048d, + 0.65119d)] [InlineData(0.57347d, 0.57483d, 0.57598d, 0.57681d, 0.57724d, 0.57906d, 0.57944d, 0.58182d, 0.58261d, 0.58300d, 0.58468d, 0.59045d, 0.59217d)] public void OneSample_Around_Two_CPU_Cycle_Method(params double[] distribution) { - var zeroMeasurement = ZeroMeasurementHelper.CheckZeroMeasurementOneSample(distribution, ThresholdMock); - Assert.False(zeroMeasurement); + Assert.True(ZeroMeasurementHelper.IsNoticeable(new Sample(distribution), ThresholdMock)); } #endregion #region TwoSamplesTests + /* Data inspired by distribution of benchmark (Workload Actual and Overhead Actual measurements) @@ -115,14 +115,21 @@ public static IEnumerable LessThanOneCycleTwoSamples { return new[] { - new object[] { new[] { 2.0037, 2.0062, 2.0066, 2.0073, 2.0089, 2.0103, 2.013, 2.0169, 2.0197, 2.0224, 2.0243, 2.0271, 2.0281, 2.0514, 2.0517 }, - new[] { 2.0426, 2.046, 2.0471, 2.0506, 2.0508, 2.0555, 2.0573, 2.0653, 2.0657, 2.0659, 2.0692, 2.0717, 2.0777, 2.0856, 2.0868 } }, - - new object[] { new[] { 2.0186, 2.0196, 2.0207, 2.0208, 2.0208, 2.0211, 2.0213, 2.0215, 2.0288, 2.0315, 2.0326, 2.039, 2.049, 2.055, 2.0598 }, - new[] { 2.0151, 2.0192, 2.0226, 2.0248, 2.0271, 2.0276, 2.0298, 2.0339, 2.0411, 2.0429, 2.0458, 2.0501, 2.061, 2.0733, 2.0744 } }, - - new object[] { new[] { 2.0049, 2.0141, 2.0194, 2.0253, 2.0264, 2.0296, 2.0333, 2.0422, 2.0438, 2.044, 2.047, 2.048, 2.0494, 2.0549, 2.0675 }, - new[] { 1.9963, 2.0037, 2.0037, 2.0046, 2.0051, 2.007, 2.0166, 2.021, 2.0225, 2.0247, 2.0391, 2.0473, 2.0572, 2.0576, 2.0582 } } + new object[] + { + new[] { 2.0037, 2.0062, 2.0066, 2.0073, 2.0089, 2.0103, 2.013, 2.0169, 2.0197, 2.0224, 2.0243, 2.0271, 2.0281, 2.0514, 2.0517 }, + new[] { 2.0426, 2.046, 2.0471, 2.0506, 2.0508, 2.0555, 2.0573, 2.0653, 2.0657, 2.0659, 2.0692, 2.0717, 2.0777, 2.0856, 2.0868 } + }, + + [ + new[] { 2.0186, 2.0196, 2.0207, 2.0208, 2.0208, 2.0211, 2.0213, 2.0215, 2.0288, 2.0315, 2.0326, 2.039, 2.049, 2.055, 2.0598 }, + new[] { 2.0151, 2.0192, 2.0226, 2.0248, 2.0271, 2.0276, 2.0298, 2.0339, 2.0411, 2.0429, 2.0458, 2.0501, 2.061, 2.0733, 2.0744 } + ], + + [ + new[] { 2.0049, 2.0141, 2.0194, 2.0253, 2.0264, 2.0296, 2.0333, 2.0422, 2.0438, 2.044, 2.047, 2.048, 2.0494, 2.0549, 2.0675 }, + new[] { 1.9963, 2.0037, 2.0037, 2.0046, 2.0051, 2.007, 2.0166, 2.021, 2.0225, 2.0247, 2.0391, 2.0473, 2.0572, 2.0576, 2.0582 } + ] }; } } @@ -131,8 +138,7 @@ public static IEnumerable LessThanOneCycleTwoSamples [MemberData(nameof(LessThanOneCycleTwoSamples))] public void TwoSamples_Less_Than_One_CPU_Cycle_Method(double[] workload, double[] overhead) { - var zeroMeasurement = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workload, overhead); - Assert.True(zeroMeasurement); + Assert.True(ZeroMeasurementHelper.AreIndistinguishable(new Sample(workload), new Sample(overhead))); } /* @@ -153,14 +159,41 @@ public static IEnumerable OneCycleTwoSamples { return new[] { - new object[] { new[] { 2.34763, 2.34861, 2.34872, 2.34953, 2.35002, 2.35614, 2.35650, 2.36323, 2.36941, 2.37376, 2.38491, 2.38619, 2.38657, 2.38902, 2.39455 }, - new[] { 2.05899, 2.06069, 2.06243, 2.06405, 2.06762, 2.06785, 2.06889, 2.06891, 2.06895, 2.07531, 2.08003, 2.08024, 2.08342, 2.08959 } }, - - new object[] { new[] { 2.36960, 2.37438, 2.37442, 2.38332, 2.38940, 2.39099, 2.39394, 2.39974, 2.40808, 2.41760, 2.41980, 2.42275, 2.42828, 2.42946, 2.43763 }, - new[] { 2.06486, 2.06599, 2.07205, 2.07660, 2.07810, 2.07841, 2.08107, 2.08714, 2.10467, 2.10469, 2.11713, 2.12078, 2.12476, 2.12858, 2.13760 } }, - - new object[] { new[] { 2.35046, 2.35630, 2.35788, 2.35801, 2.36632, 2.36841, 2.36925, 2.36980, 2.36998, 2.37153, 2.37330, 2.38491, 2.38732, 2.38853, 2.41052 }, - new[] { 2.06291, 2.06545, 2.06763, 2.07381, 2.07568, 2.07810, 2.07894, 2.08153, 2.08264, 2.09000, 2.09814, 2.10082, 2.10107, 2.10576, 2.12841 } } + new object[] + { + new[] + { + 2.34763, 2.34861, 2.34872, 2.34953, 2.35002, 2.35614, 2.35650, 2.36323, 2.36941, 2.37376, 2.38491, 2.38619, 2.38657, 2.38902, + 2.39455 + }, + new[] { 2.05899, 2.06069, 2.06243, 2.06405, 2.06762, 2.06785, 2.06889, 2.06891, 2.06895, 2.07531, 2.08003, 2.08024, 2.08342, 2.08959 } + }, + + [ + new[] + { + 2.36960, 2.37438, 2.37442, 2.38332, 2.38940, 2.39099, 2.39394, 2.39974, 2.40808, 2.41760, 2.41980, 2.42275, 2.42828, 2.42946, + 2.43763 + }, + new[] + { + 2.06486, 2.06599, 2.07205, 2.07660, 2.07810, 2.07841, 2.08107, 2.08714, 2.10467, 2.10469, 2.11713, 2.12078, 2.12476, 2.12858, + 2.13760 + } + ], + + [ + new[] + { + 2.35046, 2.35630, 2.35788, 2.35801, 2.36632, 2.36841, 2.36925, 2.36980, 2.36998, 2.37153, 2.37330, 2.38491, 2.38732, 2.38853, + 2.41052 + }, + new[] + { + 2.06291, 2.06545, 2.06763, 2.07381, 2.07568, 2.07810, 2.07894, 2.08153, 2.08264, 2.09000, 2.09814, 2.10082, 2.10107, 2.10576, + 2.12841 + } + ] }; } } @@ -169,8 +202,7 @@ public static IEnumerable OneCycleTwoSamples [MemberData(nameof(OneCycleTwoSamples))] public void TwoSamples_Around_One_CPU_Cycle_Method(double[] workload, double[] overhead) { - var zeroMeasurement = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workload, overhead); - Assert.False(zeroMeasurement); + Assert.True(ZeroMeasurementHelper.AreDistinguishable(workload, overhead)); } /* @@ -187,14 +219,19 @@ public static IEnumerable TwoCycleTwoSamples { return new[] { - new object[] { new[] { 2.6561, 2.658, 2.6606, 2.6621, 2.6636, 2.6639, 2.6656, 2.6673, 2.6741, 2.6754, 2.6787, 2.6935, 2.6997, 2.7313, 2.7394 }, - new[] { 2.0387, 2.0436, 2.0485, 2.0485, 2.0525, 2.0584, 2.0638, 2.0678, 2.0678, 2.0698, 2.0703, 2.0715, 2.0825, 2.0864 } }, - - new object[] { new[] { 2.6368, 2.6504, 2.652, 2.6541, 2.6607, 2.6642, 2.6698, 2.6749, 2.679, 2.6847, 2.6858, 2.6883, 2.6929, 2.702, 2.7134 }, - new[] { 2.04, 2.0461, 2.0481, 2.0485, 2.0502, 2.0523, 2.0547, 2.0602, 2.0613, 2.0677, 2.069, 2.0691, 2.0699, 2.0865 } }, - - new object[] { new[] { 2.6607, 2.6691, 2.67, 2.6741, 2.6753, 2.679, 2.6856, 2.6869, 2.6893, 2.692, 2.6971, 2.7146, 2.7245, 2.7368 }, - new[] { 2.0346, 2.054, 2.0655, 2.0673, 2.0718, 2.0748, 2.0766, 2.0839, 2.0856, 2.0869, 2.0924, 2.0968, 2.1129, 2.1148, 2.1328 } } + new object[] + { + new[] { 2.6561, 2.658, 2.6606, 2.6621, 2.6636, 2.6639, 2.6656, 2.6673, 2.6741, 2.6754, 2.6787, 2.6935, 2.6997, 2.7313, 2.7394 }, + new[] { 2.0387, 2.0436, 2.0485, 2.0485, 2.0525, 2.0584, 2.0638, 2.0678, 2.0678, 2.0698, 2.0703, 2.0715, 2.0825, 2.0864 } + }, + [ + new[] { 2.6368, 2.6504, 2.652, 2.6541, 2.6607, 2.6642, 2.6698, 2.6749, 2.679, 2.6847, 2.6858, 2.6883, 2.6929, 2.702, 2.7134 }, + new[] { 2.04, 2.0461, 2.0481, 2.0485, 2.0502, 2.0523, 2.0547, 2.0602, 2.0613, 2.0677, 2.069, 2.0691, 2.0699, 2.0865 } + ], + [ + new[] { 2.6607, 2.6691, 2.67, 2.6741, 2.6753, 2.679, 2.6856, 2.6869, 2.6893, 2.692, 2.6971, 2.7146, 2.7245, 2.7368 }, + new[] { 2.0346, 2.054, 2.0655, 2.0673, 2.0718, 2.0748, 2.0766, 2.0839, 2.0856, 2.0869, 2.0924, 2.0968, 2.1129, 2.1148, 2.1328 } + ] }; } } @@ -203,9 +240,9 @@ public static IEnumerable TwoCycleTwoSamples [MemberData(nameof(TwoCycleTwoSamples))] public void TwoSamples_Around_Two_CPU_Cycle_Method(double[] workload, double[] overhead) { - var zeroMeasurement = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workload, overhead); - Assert.False(zeroMeasurement); + Assert.True(ZeroMeasurementHelper.AreDistinguishable(workload, overhead)); } + #endregion } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesVerifyTests.cs b/tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesVerifyTests.cs index a1086da1de..8da86700c7 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesVerifyTests.cs +++ b/tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesVerifyTests.cs @@ -9,6 +9,7 @@ using BenchmarkDotNet.Tests.Mocks; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Tests.Builders; +using BenchmarkDotNet.Tests.Infra; using BenchmarkDotNet.Validators; using JetBrains.Annotations; using VerifyXunit; @@ -17,7 +18,6 @@ namespace BenchmarkDotNet.Tests.Attributes { [Collection("VerifyTests")] - [UsesVerify] public class ParamsAllValuesVerifyTests : IDisposable { private readonly CultureInfo initCulture; @@ -53,7 +53,7 @@ public Task BenchmarkShouldProduceSummary(Type benchmarkType) foreach (var error in errors) logger.WriteLineError("* " + error.Message); - var settings = VerifySettingsFactory.Create(); + var settings = VerifyHelper.Create(); settings.UseTextForParameters(benchmarkType.Name); return Verifier.Verify(logger.GetLog(), settings); } diff --git a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfBool.verified.txt b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfBool.verified.txt index 41554d0c7c..d89bd4c091 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfBool.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfBool.verified.txt @@ -2,14 +2,14 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | ParamProperty | Mean | Error | StdDev | ---------- |-------------- |---------:|--------:|--------:| - Benchmark | False | 102.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | True | 202.0 ns | 6.09 ns | 1.58 ns | ^ + Benchmark | False | 114.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | True | 214.5 ns | 5.88 ns | 8.80 ns | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfEnum.verified.txt b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfEnum.verified.txt index c1a301f759..2e8ac5702e 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfEnum.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfEnum.verified.txt @@ -2,15 +2,15 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | ParamProperty | Mean | Error | StdDev | ---------- |-------------- |---------:|--------:|--------:| - Benchmark | A | 102.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | B | 202.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | C | 302.0 ns | 6.09 ns | 1.58 ns | ^ + Benchmark | A | 114.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | B | 214.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | C | 314.5 ns | 5.88 ns | 8.80 ns | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableBool.verified.txt b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableBool.verified.txt index 61f4eeed4f..d292990cb8 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableBool.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableBool.verified.txt @@ -2,15 +2,15 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | ParamProperty | Mean | Error | StdDev | ---------- |-------------- |---------:|--------:|--------:| - Benchmark | ? | 102.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | False | 202.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | True | 302.0 ns | 6.09 ns | 1.58 ns | ^ + Benchmark | ? | 114.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | False | 214.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | True | 314.5 ns | 5.88 ns | 8.80 ns | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableEnum.verified.txt b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableEnum.verified.txt index 0c1dd08fdc..32f13c0208 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableEnum.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableEnum.verified.txt @@ -2,16 +2,16 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | ParamProperty | Mean | Error | StdDev | ---------- |-------------- |---------:|--------:|--------:| - Benchmark | ? | 102.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | A | 202.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | B | 302.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | C | 402.0 ns | 6.09 ns | 1.58 ns | ^ + Benchmark | ? | 114.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | A | 214.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | B | 314.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | C | 414.5 ns | 5.88 ns | 8.80 ns | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedFlagsEnumError.verified.txt b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedFlagsEnumError.verified.txt index ba31d1bc3b..bd49ee8d35 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedFlagsEnumError.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedFlagsEnumError.verified.txt @@ -2,14 +2,14 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | ParamProperty | Mean | Error | StdDev | ---------- |-------------- |---------:|--------:|--------:| - Benchmark | 0 | 102.0 ns | 6.09 ns | 1.58 ns | + Benchmark | 0 | 114.5 ns | 5.88 ns | 8.80 ns | Errors: 1 * Unable to use TestFlagsEnum with [ParamsAllValues], because it's flags enum. diff --git a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedNullableTypeError.verified.txt b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedNullableTypeError.verified.txt index 19566395d8..aceee21a7c 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedNullableTypeError.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedNullableTypeError.verified.txt @@ -2,15 +2,15 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | ParamProperty | Mean | Error | StdDev | ---------- |-------------- |---------:|--------:|--------:| - Benchmark | ? | 102.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | 0 | 202.0 ns | 6.09 ns | 1.58 ns | ^ + Benchmark | ? | 114.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | 0 | 214.5 ns | 5.88 ns | 8.80 ns | ^ Errors: 1 * Type Int32 cannot be used with [ParamsAllValues], allowed types are: bool, enum types and nullable type for another allowed type. diff --git a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedTypeError.verified.txt b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedTypeError.verified.txt index 9a27385161..d53d688ab8 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedTypeError.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedTypeError.verified.txt @@ -2,14 +2,14 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | ParamProperty | Mean | Error | StdDev | ---------- |-------------- |---------:|--------:|--------:| - Benchmark | 0 | 102.0 ns | 6.09 ns | 1.58 ns | + Benchmark | 0 | 114.5 ns | 5.88 ns | 8.80 ns | Errors: 1 * Type Int32 cannot be used with [ParamsAllValues], allowed types are: bool, enum types and nullable type for another allowed type. diff --git a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj index 5b7a5b9cdd..b37e22deda 100755 --- a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj +++ b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj @@ -2,7 +2,7 @@ BenchmarkDotNet.Tests - net7.0;net462 + net8.0;net462 BenchmarkDotNet.Tests BenchmarkDotNet.Tests true @@ -15,18 +15,18 @@ - - - - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + @@ -34,12 +34,23 @@ + - + + + + + CpuInfoFormatterTests + CpuInfoFormatterTests.cs + + + + + diff --git a/tests/BenchmarkDotNet.Tests/BuildResultTests.cs b/tests/BenchmarkDotNet.Tests/BuildResultTests.cs index 978da2a516..beba98e49e 100644 --- a/tests/BenchmarkDotNet.Tests/BuildResultTests.cs +++ b/tests/BenchmarkDotNet.Tests/BuildResultTests.cs @@ -11,24 +11,24 @@ public class BuildResultTests [Fact] public void NotFullyCompatibleMsBuildErrorIsTranslatedToMoreUserFriendlyVersion() { - const string msbuildError = @"C:\Program Files\dotnet\sdk\3.0.100-preview9-013617\Microsoft.Common.CurrentVersion.targets(1653,5): warning NU1702: ProjectReference 'C:\Projects\BenchmarkDotNet\tests\BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetFramework\BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetFramework.csproj' was resolved using '.NETFramework,Version=v4.6.2' instead of the project target framework '.NETCoreApp,Version=v2.1'. This project may not be fully compatible with your project. [C:\Projects\BenchmarkDotNet\tests\BenchmarkDotNet.IntegrationTests\bin\Release\net462\Job-VUALUD\BenchmarkDotNet.Autogenerated.csproj]"; + const string msbuildError = @"C:\Program Files\dotnet\sdk\3.0.100-preview9-013617\Microsoft.Common.CurrentVersion.targets(1653,5): warning NU1702: ProjectReference 'C:\Projects\BenchmarkDotNet\tests\BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetFramework\BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetFramework.csproj' was resolved using '.NETFramework,Version=v4.6.2' instead of the project target framework '.NETCoreApp,Version=v8.0'. This project may not be fully compatible with your project. [C:\Projects\BenchmarkDotNet\tests\BenchmarkDotNet.IntegrationTests\bin\Release\net462\Job-VUALUD\BenchmarkDotNet.Autogenerated.csproj]"; - string expected = $@"The project which defines benchmarks does not target 'netcoreapp2.1'." + Environment.NewLine + - $"You need to add 'netcoreapp2.1' to in your project file " + + string expected = $@"The project which defines benchmarks does not target 'net8.0'." + Environment.NewLine + + $"You need to add 'net8.0' to in your project file " + @"('C:\Projects\BenchmarkDotNet\tests\BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetFramework\BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetFramework.csproj')." + Environment.NewLine + - "Example: net462;netcoreapp2.1"; + "Example: net462;net8.0"; Verify(msbuildError, true, expected); } [Fact] public void NotCompatibleMsBuildErrorIsTranslatedToMoreUserFriendlyVersion() { - const string msbuildError = @"error NU1201: Project BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetCore is not compatible with net462 (.NETFramework,Version=v4.6.2) / win7-x64. Project BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetCore supports: netcoreapp2.1 (.NETCoreApp,Version=v2.1)"; + const string msbuildError = @"error NU1201: Project BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetCore is not compatible with net462 (.NETFramework,Version=v4.6.2) / win7-x64. Project BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetCore supports: net8.0 (.NETCoreApp,Version=v8.0)"; string expected = $@"The project which defines benchmarks does not target 'net462'." + Environment.NewLine + $"You need to add 'net462' to in your project file " + @"('BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetCore')." + Environment.NewLine + - "Example: netcoreapp2.1;net462"; + "Example: net8.0;net462"; Verify(msbuildError, true, expected); } @@ -36,7 +36,7 @@ public void NotCompatibleMsBuildErrorIsTranslatedToMoreUserFriendlyVersion() [Fact] public void MissingSdkIsTranslatedToMoreUserFriendlyVersion() { - const string msbuildError = @"C:\Program Files\dotnet\sdk\3.0.100-preview9-013617\Sdks\Microsoft.NET.Sdk\targets\Microsoft.NET.TargetFrameworkInference.targets(134,5): error NETSDK1045: The current .NET SDK does not support targeting .NET Core 5.0. Either target .NET Core 3.0 or lower, or use a version of the .NET SDK that supports .NET Core 5.0. [C:\Projects\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\netcoreapp2.1\bad57fca-694a-41ad-b630-9e5317a782ab\BenchmarkDotNet.Autogenerated.csproj]"; + const string msbuildError = @"C:\Program Files\dotnet\sdk\3.0.100-preview9-013617\Sdks\Microsoft.NET.Sdk\targets\Microsoft.NET.TargetFrameworkInference.targets(134,5): error NETSDK1045: The current .NET SDK does not support targeting .NET Core 5.0. Either target .NET Core 3.0 or lower, or use a version of the .NET SDK that supports .NET Core 5.0. [C:\Projects\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\net8.0\bad57fca-694a-41ad-b630-9e5317a782ab\BenchmarkDotNet.Autogenerated.csproj]"; string expected = "The current .NET SDK does not support targeting .NET Core 5.0. You need to install it or pass the path to dotnet cli via the `--cli` console line argument."; diff --git a/tests/BenchmarkDotNet.Tests/Builders/HostEnvironmentInfoBuilder.cs b/tests/BenchmarkDotNet.Tests/Builders/HostEnvironmentInfoBuilder.cs index d22256f0a9..29d690892b 100644 --- a/tests/BenchmarkDotNet.Tests/Builders/HostEnvironmentInfoBuilder.cs +++ b/tests/BenchmarkDotNet.Tests/Builders/HostEnvironmentInfoBuilder.cs @@ -1,11 +1,14 @@ using System; +using System.Diagnostics.CodeAnalysis; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Portability.Cpu; using Perfolizer.Horology; +using Perfolizer.Models; namespace BenchmarkDotNet.Tests.Builders { + [SuppressMessage("ReSharper", "InconsistentNaming")] public class HostEnvironmentInfoBuilder { private string architecture = "64mock"; @@ -20,16 +23,18 @@ public class HostEnvironmentInfoBuilder private bool isServerGC = false; private string jitInfo = "RyuJIT-v4.6.x.mock"; private string jitModules = "clrjit-v4.6.x.mock"; - private string osVersion = "Microsoft Windows NT 10.0.x.mock"; + private OsInfo os = new() { Display = "Microsoft Windows NT 10.0.x.mock" }; private string runtimeVersion = "Clr 4.0.x.mock"; - private CpuInfo cpuInfo = new CpuInfo("MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", - physicalProcessorCount: 1, - physicalCoreCount: 4, - logicalCoreCount: 8, - nominalFrequency: Frequency.FromMHz(3100), - maxFrequency: Frequency.FromMHz(3100), - minFrequency: Frequency.FromMHz(3100)); + private readonly CpuInfo cpu = new() + { + ProcessorName = "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 4, + LogicalCoreCount = 8, + NominalFrequencyHz = Frequency.FromMHz(3100).Hertz.RoundToLong(), + MaxFrequencyHz = Frequency.FromMHz(3100).Hertz.RoundToLong() + }; private VirtualMachineHypervisor? virtualMachineHypervisor = HyperV.Default; @@ -55,7 +60,7 @@ public HostEnvironmentInfo Build() { return new MockHostEnvironmentInfo(architecture, benchmarkDotNetVersion, chronometerFrequency, configuration, dotNetSdkVersion, hardwareTimerKind, hasAttachedDebugger, hasRyuJit, isConcurrentGC, isServerGC, - jitInfo, jitModules, osVersion, cpuInfo, runtimeVersion, virtualMachineHypervisor); + jitInfo, jitModules, os, cpu, runtimeVersion, virtualMachineHypervisor); } } @@ -64,7 +69,7 @@ internal class MockHostEnvironmentInfo : HostEnvironmentInfo public MockHostEnvironmentInfo( string architecture, string benchmarkDotNetVersion, Frequency chronometerFrequency, string configuration, string dotNetSdkVersion, HardwareTimerKind hardwareTimerKind, bool hasAttachedDebugger, bool hasRyuJit, bool isConcurrentGC, bool isServerGC, - string jitInfo, string jitModules, string osVersion, CpuInfo cpuInfo, + string jitInfo, string jitModules, OsInfo os, CpuInfo cpu, string runtimeVersion, VirtualMachineHypervisor virtualMachineHypervisor) { Architecture = architecture; @@ -79,8 +84,8 @@ public MockHostEnvironmentInfo( IsServerGC = isServerGC; JitInfo = jitInfo; HardwareIntrinsicsShort = ""; - OsVersion = new Lazy(() => osVersion); - CpuInfo = new Lazy(() => cpuInfo); + Os = new Lazy(() => os); + Cpu = new Lazy(() => cpu); RuntimeVersion = runtimeVersion; VirtualMachineHypervisor = new Lazy(() => virtualMachineHypervisor); } diff --git a/tests/BenchmarkDotNet.Tests/Builders/VerifySettingsFactory.cs b/tests/BenchmarkDotNet.Tests/Builders/VerifySettingsFactory.cs deleted file mode 100644 index 027b5272a7..0000000000 --- a/tests/BenchmarkDotNet.Tests/Builders/VerifySettingsFactory.cs +++ /dev/null @@ -1,15 +0,0 @@ -using VerifyTests; - -namespace BenchmarkDotNet.Tests.Builders -{ - public static class VerifySettingsFactory - { - public static VerifySettings Create() - { - var result = new VerifySettings(); - result.UseDirectory("VerifiedFiles"); - result.DisableDiff(); - return result; - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Columns/MetricColumnTests.cs b/tests/BenchmarkDotNet.Tests/Columns/MetricColumnTests.cs index f16c4bd7f6..4c7678ba67 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/MetricColumnTests.cs +++ b/tests/BenchmarkDotNet.Tests/Columns/MetricColumnTests.cs @@ -11,6 +11,7 @@ using BenchmarkDotNet.Parameters; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Mocks; using BenchmarkDotNet.Validators; using Perfolizer.Horology; using Xunit; @@ -37,7 +38,7 @@ private static Summary CreateMockSummary(bool printUnitsInContent, bool printUni var summaryStyle = new SummaryStyle(TestCultureInfo.Instance, printUnitsInHeader, null, timeUnit, printUnitsInContent); var config = new ManualConfig().WithSummaryStyle(summaryStyle); var benchmarkCase = new BenchmarkCase( - new Descriptor(null, null), + new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo), Job.Dry, new ParameterInstances(ImmutableArray.Empty), ImmutableConfigBuilder.Create(config)); diff --git a/tests/BenchmarkDotNet.Tests/Columns/StatisticalTestColumnTests.cs b/tests/BenchmarkDotNet.Tests/Columns/StatisticalTestColumnTests.cs index ee45b61347..7a0d4bb1fd 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/StatisticalTestColumnTests.cs +++ b/tests/BenchmarkDotNet.Tests/Columns/StatisticalTestColumnTests.cs @@ -2,9 +2,9 @@ using System.Linq; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Mathematics; -using Perfolizer.Mathematics.SignificanceTesting; -using Perfolizer.Mathematics.Thresholds; using BenchmarkDotNet.Reports; +using Perfolizer.Mathematics.Distributions.ContinuousDistributions; +using Perfolizer.Metrology; using Xunit; namespace BenchmarkDotNet.Tests.Columns @@ -12,60 +12,56 @@ namespace BenchmarkDotNet.Tests.Columns public class StatisticalTestColumnTests { [Theory] - [InlineData(StatisticalTestKind.MannWhitney, ThresholdUnit.Ratio, 0.05)] - [InlineData(StatisticalTestKind.Welch, ThresholdUnit.Ratio, 0.05)] - [InlineData(StatisticalTestKind.Welch, ThresholdUnit.Milliseconds, 10)] - [InlineData(StatisticalTestKind.MannWhitney, ThresholdUnit.Milliseconds, 10)] - public void NoDifferenceIfValuesAreTheSame(StatisticalTestKind statisticalTestKind, ThresholdUnit thresholdUnit, double thresholdValue) + [InlineData("5%")] + [InlineData("10ms")] + public void NoDifferenceIfValuesAreTheSame(string threshold) { - var values = Enumerable.Repeat(100.0, 20).ToArray(); - - Compare(statisticalTestKind, thresholdUnit, thresholdValue, values, values, "Base"); + double[] values = Enumerable.Repeat(100.0, 20).ToArray(); + Compare(threshold, values, values, "Baseline"); } [Theory] - [InlineData(StatisticalTestKind.MannWhitney, ThresholdUnit.Ratio, 0.02)] - [InlineData(StatisticalTestKind.Welch, ThresholdUnit.Ratio, 0.02)] - public void RegressionsAreDetected(StatisticalTestKind statisticalTestKind, ThresholdUnit thresholdUnit, double thresholdValue) + [InlineData("2%")] + public void RegressionsAreDetected(string threshold) { - var baseline = new[] { 10.0, 10.01, 10.02, 10.0, 10.03, 10.02, 9.99, 9.98, 10.0, 10.02 }; - var current = baseline.Select(value => value * 1.03).ToArray(); + double[]? baseline = [10.0, 10.01, 10.02, 10.0, 10.03, 10.02, 9.99, 9.98, 10.0, 10.02]; + double[]? current = baseline.Select(value => value * 1.03).ToArray(); - Compare(statisticalTestKind, thresholdUnit, thresholdValue, baseline, current, "Slower"); + Compare(threshold, baseline, current, "Slower"); } [Theory] - [InlineData(StatisticalTestKind.MannWhitney, ThresholdUnit.Ratio, 0.02)] - [InlineData(StatisticalTestKind.Welch, ThresholdUnit.Ratio, 0.02)] - public void CanCompareDifferentSampleSizes(StatisticalTestKind statisticalTestKind, ThresholdUnit thresholdUnit, double thresholdValue) + [InlineData("2%")] + public void CanCompareDifferentSampleSizes(string threshold) { - var baseline = new[] { 10.0, 10.01, 10.02, 10.0, 10.03, 10.02, 9.99, 9.98, 10.0, 10.02 }; - var current = baseline + double[] baseline = new NormalDistribution(10, 0.01).Random(1729).Next(30); + double[] current = baseline .Skip(1) // we skip one element to make sure the sample size is different .Select(value => value * 1.03).ToArray(); - Compare(statisticalTestKind, thresholdUnit, thresholdValue, baseline, current, "Slower"); + Compare(threshold, baseline, current, "Slower"); } [Theory] - [InlineData(StatisticalTestKind.MannWhitney, ThresholdUnit.Ratio, 0.02)] - [InlineData(StatisticalTestKind.Welch, ThresholdUnit.Ratio, 0.02)] - public void ImprovementsDetected(StatisticalTestKind statisticalTestKind, ThresholdUnit thresholdUnit, double thresholdValue) + [InlineData("2%")] + public void ImprovementsDetected(string threshold) { var baseline = new[] { 10.0, 10.01, 10.02, 10.0, 10.03, 10.02, 9.99, 9.98, 10.0, 10.02 }; var current = baseline.Select(value => value * 0.97).ToArray(); - Compare(statisticalTestKind, thresholdUnit, thresholdValue, baseline, current, "Faster"); + Compare(threshold, baseline, current, "Faster"); } - private static void Compare(StatisticalTestKind statisticalTestKind, ThresholdUnit thresholdUnit, double thresholdValue, double[] baseline, double[] current, string expectedResult) + private static void Compare(string threshold, double[] baseline, double[] current, string expectedResult) { - var sut = new StatisticalTestColumn(statisticalTestKind, Threshold.Create(thresholdUnit, thresholdValue)); + var sut = new StatisticalTestColumn(Threshold.Parse(threshold)); var emptyMetrics = new Dictionary(); - Assert.Equal(expectedResult, sut.GetValue(null, null, new Statistics(baseline), emptyMetrics, new Statistics(current), emptyMetrics, isBaseline: true)); - Assert.Equal(expectedResult, sut.GetValue(null, null, new Statistics(baseline), emptyMetrics, new Statistics(current), emptyMetrics, isBaseline: false)); + Assert.Equal(expectedResult, + sut.GetValue(null, null, new Statistics(baseline), emptyMetrics, new Statistics(current), emptyMetrics, isBaseline: true)); + Assert.Equal(expectedResult, + sut.GetValue(null, null, new Statistics(baseline), emptyMetrics, new Statistics(current), emptyMetrics, isBaseline: false)); } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Common/AbsoluteEqualityComparer.cs b/tests/BenchmarkDotNet.Tests/Common/AbsoluteEqualityComparer.cs new file mode 100644 index 0000000000..80a477d348 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Common/AbsoluteEqualityComparer.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; + +namespace BenchmarkDotNet.Tests.Common; + +public class AbsoluteEqualityComparer(double eps) : IEqualityComparer +{ + public static readonly IEqualityComparer E4 = new AbsoluteEqualityComparer(1e-4); + public static readonly IEqualityComparer E5 = new AbsoluteEqualityComparer(1e-5); + public static readonly IEqualityComparer E9 = new AbsoluteEqualityComparer(1e-9); + + public bool Equals(double x, double y) + { + if (double.IsPositiveInfinity(x) && double.IsPositiveInfinity(y)) + return true; + if (double.IsNegativeInfinity(x) && double.IsNegativeInfinity(y)) + return true; + if (double.IsNaN(x) && double.IsNaN(y)) + return true; + return Math.Abs(x - y) < eps; + } + + public int GetHashCode(double x) => x.GetHashCode(); +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs index ec9b85021f..dd25527e8a 100644 --- a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs +++ b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs @@ -23,8 +23,6 @@ using Xunit.Abstractions; using BenchmarkDotNet.Portability; using Perfolizer.Horology; -using Perfolizer.Mathematics.SignificanceTesting; -using Perfolizer.Mathematics.Thresholds; namespace BenchmarkDotNet.Tests { @@ -76,7 +74,7 @@ public void UserCanSpecifyHowManyTimesTheBenchmarkShouldBeExecuted() var config = ConfigParser.Parse(new[] { "--LaunchCount", launchCount.ToString(), - "--warmupCount", warmupCount.ToString(), + "--warmupCount", warmupCount.ToString(), "--iterationTime", iterationTime.ToString(), "--iterationCount", iterationCount.ToString() }, new OutputLogger(Output)).config; @@ -141,12 +139,14 @@ public void CoreRunConfigParsedCorrectlyWhenRuntimeNotSpecified() var fakeDotnetCliPath = typeof(object).Assembly.Location; var fakeCoreRunPath = typeof(ConfigParserTests).Assembly.Location; var fakeRestorePackages = Path.GetTempPath(); - var config = ConfigParser.Parse(new[] { "--job=Dry", "--coreRun", fakeCoreRunPath, "--cli", fakeDotnetCliPath, "--packages", fakeRestorePackages }, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "--job=Dry", "--coreRun", fakeCoreRunPath, "--cli", fakeDotnetCliPath, "--packages", fakeRestorePackages }, + new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); CoreRunToolchain toolchain = config.GetJobs().Single().GetToolchain() as CoreRunToolchain; Assert.NotNull(toolchain); - Assert.Equal(RuntimeInformation.GetCurrentRuntime().MsBuildMoniker, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); // runtime was not specified so the current was used + Assert.Equal(RuntimeInformation.GetCurrentRuntime().MsBuildMoniker, + ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); // runtime was not specified so the current was used Assert.Equal(fakeCoreRunPath, toolchain.SourceCoreRun.FullName); Assert.Equal(fakeDotnetCliPath, toolchain.CustomDotNetCliPath.FullName); Assert.Equal(fakeRestorePackages, toolchain.RestorePath.FullName); @@ -162,17 +162,19 @@ public void SpecifyingCoreRunWithFullFrameworkTargetsMostRecentTfm() CoreRunToolchain coreRunToolchain = (CoreRunToolchain)coreRunJob.GetToolchain(); DotNetCliGenerator generator = (DotNetCliGenerator)coreRunToolchain.Generator; - Assert.Equal("net8.0", generator.TargetFrameworkMoniker); + Assert.Equal("net10.0", generator.TargetFrameworkMoniker); } [FactEnvSpecific("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process", EnvRequirement.DotNetCoreOnly)] public void SpecifyingCoreRunAndRuntimeCreatesTwoJobs() { - const string runtime = "net7.0"; + const string runtime = "net8.0"; var fakeDotnetCliPath = typeof(object).Assembly.Location; var fakeCoreRunPath = typeof(ConfigParserTests).Assembly.Location; var fakeRestorePackages = Path.GetTempPath(); - var config = ConfigParser.Parse(new[] { "--job=Dry", "--coreRun", fakeCoreRunPath, "--cli", fakeDotnetCliPath, "--packages", fakeRestorePackages, "-r", runtime }, new OutputLogger(Output)).config; + var config = ConfigParser + .Parse(new[] { "--job=Dry", "--coreRun", fakeCoreRunPath, "--cli", fakeDotnetCliPath, "--packages", fakeRestorePackages, "-r", runtime }, + new OutputLogger(Output)).config; Assert.Equal(2, config.GetJobs().Count()); @@ -231,9 +233,8 @@ public void UserCanSpecifyMultipleCoreRunPaths() var jobs = config.GetJobs().ToArray(); Assert.Equal(2, jobs.Length); - Assert.Single(jobs.Where(job => job.GetToolchain() is CoreRunToolchain toolchain && toolchain.SourceCoreRun.FullName == fakeCoreRunPath_1)); - Assert.Single(jobs.Where(job => job.GetToolchain() is CoreRunToolchain toolchain && toolchain.SourceCoreRun.FullName == fakeCoreRunPath_2)); - Assert.Equal(2, jobs.Select(job => job.Id).Distinct().Count()); // each job must have a unique ID + Assert.Single(jobs, job => job.GetToolchain() is CoreRunToolchain toolchain && toolchain.SourceCoreRun.FullName == fakeCoreRunPath_1); + Assert.Single(jobs, job => job.GetToolchain() is CoreRunToolchain toolchain && toolchain.SourceCoreRun.FullName == fakeCoreRunPath_2); } [Fact] @@ -243,7 +244,7 @@ public void MonoPathParsedCorrectly() var config = ConfigParser.Parse(new[] { "-r", "mono", "--monoPath", fakeMonoPath }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is MonoRuntime mono && mono.CustomPath == fakeMonoPath)); + Assert.Single(config.GetJobs(), job => job.Environment.Runtime is MonoRuntime mono && mono.CustomPath == fakeMonoPath); } [FactEnvSpecific("Testing local builds of Full .NET Framework is supported only on Windows", EnvRequirement.WindowsOnly)] @@ -253,13 +254,13 @@ public void ClrVersionParsedCorrectly() var config = ConfigParser.Parse(new[] { "--clrVersion", clrVersion }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is ClrRuntime clr && clr.Version == clrVersion)); + Assert.Single(config.GetJobs(), job => job.Environment.Runtime is ClrRuntime clr && clr.Version == clrVersion); } [Fact] public void IlCompilerPathParsedCorrectly() { - var fakePath = new FileInfo(typeof(ConfigParserTests).Assembly.Location).Directory; + var fakePath = new FileInfo(typeof(ConfigParserTests).Assembly.Location).Directory; var config = ConfigParser.Parse(new[] { "-r", "nativeaot60", "--ilcPackages", fakePath.FullName }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); @@ -274,6 +275,12 @@ public void IlCompilerPathParsedCorrectly() [InlineData("netcoreapp2.1", true)] [InlineData("netcoreapp2.2", true)] [InlineData("netcoreapp3.0", true)] + [InlineData("netcoreapp3.1", true)] + [InlineData("net5.0", true)] + [InlineData("net6.0", true)] + [InlineData("net7.0", true)] + [InlineData("net8.0", true)] + [InlineData("net9.0", true)] [InlineData("net462", false)] [InlineData("net48", false)] public void DotNetCliParsedCorrectly(string tfm, bool isCore) @@ -286,12 +293,12 @@ public void DotNetCliParsedCorrectly(string tfm, bool isCore) if (isCore) { Assert.True(toolchain is CsProjCoreToolchain); - Assert.Equal(fakeDotnetCliPath, ((CsProjCoreToolchain) toolchain).CustomDotNetCliPath); + Assert.Equal(fakeDotnetCliPath, ((CsProjCoreToolchain)toolchain).CustomDotNetCliPath); } else { Assert.True(toolchain is CsProjClassicNetToolchain); - Assert.Equal(fakeDotnetCliPath, ((CsProjClassicNetToolchain) toolchain).CustomDotNetCliPath); + Assert.Equal(fakeDotnetCliPath, ((CsProjClassicNetToolchain)toolchain).CustomDotNetCliPath); } Assert.Equal(tfm, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); } @@ -301,7 +308,7 @@ public void DotNetCliParsedCorrectly(string tfm, bool isCore) [InlineData(ConfigOptions.KeepBenchmarkFiles, "--keepFiles")] [InlineData(ConfigOptions.DontOverwriteResults, "--noOverwrite")] [InlineData(ConfigOptions.StopOnFirstError, "--stopOnFirstError")] - [InlineData(ConfigOptions.DisableLogFile, "--disableLogFile" )] + [InlineData(ConfigOptions.DisableLogFile, "--disableLogFile")] [InlineData(ConfigOptions.LogBuildOutput, "--logBuildOutput")] [InlineData(ConfigOptions.GenerateMSBuildBinLog | ConfigOptions.KeepBenchmarkFiles, "--generateBinLog")] [InlineData( @@ -332,7 +339,7 @@ public void WhenConfigOptionsFlagsAreNotSpecifiedTheyAreNotSet() public void PackagesPathParsedCorrectly() { var fakeRestoreDirectory = new FileInfo(typeof(object).Assembly.Location).Directory.FullName; - var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.0", "--packages", fakeRestoreDirectory }, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.1", "--packages", fakeRestoreDirectory }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); CsProjCoreToolchain toolchain = config.GetJobs().Single().GetToolchain() as CsProjCoreToolchain; @@ -344,7 +351,7 @@ public void PackagesPathParsedCorrectly() public void UserCanSpecifyBuildTimeout() { const int timeoutInSeconds = 10; - var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.0", "--buildTimeout", timeoutInSeconds.ToString() }, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.1", "--buildTimeout", timeoutInSeconds.ToString() }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); CsProjCoreToolchain toolchain = config.GetJobs().Single().GetToolchain() as CsProjCoreToolchain; @@ -355,7 +362,7 @@ public void UserCanSpecifyBuildTimeout() [Fact] public void WhenUserDoesNotSpecifyTimeoutTheDefaultValueIsUsed() { - var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.0" }, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.1" }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); CsProjCoreToolchain toolchain = config.GetJobs().Single().GetToolchain() as CsProjCoreToolchain; @@ -363,6 +370,22 @@ public void WhenUserDoesNotSpecifyTimeoutTheDefaultValueIsUsed() Assert.Equal(DefaultConfig.Instance.BuildTimeout, config.BuildTimeout); } + [Fact] + public void UserCanSpecifyWakeLock() + { + var config = ConfigParser.Parse(["--wakeLock", "Display"], new OutputLogger(Output)).config; + + Assert.Equal(WakeLockType.Display, config.WakeLock); + } + + [Fact] + public void WhenUserDoesNotSpecifyWakeLockTheDefaultValueIsUsed() + { + var config = ConfigParser.Parse([], new OutputLogger(Output)).config; + + Assert.Equal(DefaultConfig.Instance.WakeLock, config.WakeLock); + } + [Theory] [InlineData("net461")] [InlineData("net462")] @@ -383,10 +406,17 @@ public void NetFrameworkMonikerParsedCorrectly(string tfm) [Theory] [InlineData("net50")] + [InlineData("net5.0")] [InlineData("net60")] + [InlineData("net6.0")] [InlineData("net70")] + [InlineData("net7.0")] [InlineData("net80")] + [InlineData("net8.0")] [InlineData("net90")] + [InlineData("net9.0")] + [InlineData("net10_0")] + [InlineData("net10.0")] public void NetMonikersAreRecognizedAsNetCoreMonikers(string tfm) { var config = ConfigParser.Parse(new[] { "-r", tfm }, new OutputLogger(Output)).config; @@ -413,56 +443,63 @@ public void PlatformSpecificMonikersAreSupported(string msBuildMoniker) [Fact] public void CanCompareFewDifferentRuntimes() { - var config = ConfigParser.Parse(new[] { "--runtimes", "net462", "MONO", "netcoreapp3.0", "nativeaot6.0", "nativeAOT7.0", "nativeAOT8.0" }, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(["--runtimes", "net462", "MONO", "netcoreapp2.0", "nativeaot6.0", "nativeAOT7.0", "nativeAOT8.0"], + new OutputLogger(Output)).config; Assert.True(config.GetJobs().First().Meta.Baseline); // when the user provides multiple runtimes the first one should be marked as baseline - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is ClrRuntime clrRuntime && clrRuntime.MsBuildMoniker == "net462")); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is MonoRuntime)); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is CoreRuntime coreRuntime && coreRuntime.MsBuildMoniker == "netcoreapp3.0" && coreRuntime.RuntimeMoniker == RuntimeMoniker.NetCoreApp30)); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net6.0" && nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot60)); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net7.0" && nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot70)); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net8.0" && nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot80)); + Assert.Single(config.GetJobs(), job => job.Environment.Runtime is ClrRuntime clrRuntime && clrRuntime.MsBuildMoniker == "net462"); + Assert.Single(config.GetJobs(), job => job.Environment.Runtime is MonoRuntime); + Assert.Single(config.GetJobs(), job => + job.Environment.Runtime is CoreRuntime coreRuntime && coreRuntime.MsBuildMoniker == "netcoreapp2.0" && + coreRuntime.RuntimeMoniker == RuntimeMoniker.NetCoreApp20); + Assert.Single(config.GetJobs(), job => + job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net6.0" && + nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot60); + Assert.Single(config.GetJobs(), job => + job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net7.0" && + nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot70); + Assert.Single(config.GetJobs(), job => + job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net8.0" && + nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot80); } [Theory] - [InlineData(ThresholdUnit.Ratio, 5)] - [InlineData(ThresholdUnit.Milliseconds, 10)] - public void CanUseStatisticalTestsToCompareFewDifferentRuntimes(ThresholdUnit thresholdUnit, double thresholdValue) + [InlineData("5%")] + [InlineData("10ms")] + public void CanUseStatisticalTestsToCompareFewDifferentRuntimes(string threshold) { - var config = ConfigParser.Parse(new[] - { - "--runtimes", "netcoreapp2.1", "netcoreapp2.2", - "--statisticalTest", $"{thresholdValue.ToString(CultureInfo.InvariantCulture)}{thresholdUnit.ToShortName()}" - }, new OutputLogger(Output)).config; + string[] arguments = ["--runtimes", "net6.0", "net8.0", "--statisticalTest", threshold]; + var config = ConfigParser.Parse(arguments, new OutputLogger(Output)).config; var mockSummary = MockFactory.CreateSummary(config); Assert.True(config.GetJobs().First().Meta.Baseline); // when the user provides multiple runtimes the first one should be marked as baseline Assert.False(config.GetJobs().Last().Meta.Baseline); - var statisticalTestColumn = config.GetColumnProviders().SelectMany(columnProvider => columnProvider.GetColumns(mockSummary)).OfType().Single(); + var statisticalTestColumn = config.GetColumnProviders().SelectMany(columnProvider => columnProvider.GetColumns(mockSummary)) + .OfType().Single(); - Assert.Equal(StatisticalTestKind.MannWhitney, statisticalTestColumn.Kind); - Assert.Equal(Threshold.Create(thresholdUnit, thresholdUnit == ThresholdUnit.Ratio ? thresholdValue / 100.0 : thresholdValue), statisticalTestColumn.Threshold); + Assert.Equal(threshold, statisticalTestColumn.Threshold.ToString()); } [Fact] public void SpecifyingInvalidStatisticalTestsThresholdMeansFailure() { - Assert.False(ConfigParser.Parse(new[] {"--statisticalTest", "not a number" }, new OutputLogger(Output)).isSuccess); - Assert.False(ConfigParser.Parse(new[] {"--statisticalTest", "1unknownUnit" }, new OutputLogger(Output)).isSuccess); - Assert.False(ConfigParser.Parse(new[] {"--statisticalTest", "1 unknownUnit" }, new OutputLogger(Output)).isSuccess); - Assert.False(ConfigParser.Parse(new[] {"--statisticalTest", "%1" }, new OutputLogger(Output)).isSuccess); // reverse order - a typo + Assert.False(ConfigParser.Parse(["--statisticalTest", "not a number"], new OutputLogger(Output)).isSuccess); + Assert.False(ConfigParser.Parse(["--statisticalTest", "1unknownUnit"], new OutputLogger(Output)).isSuccess); + Assert.False(ConfigParser.Parse(["--statisticalTest", "1 unknownUnit"], new OutputLogger(Output)).isSuccess); + Assert.False(ConfigParser.Parse(["--statisticalTest", "%1"], new OutputLogger(Output)).isSuccess); // reverse order - a typo } [Fact] public void CanParseHardwareCounters() { - var config = ConfigParser.Parse(new[] { "--counters", $"{nameof(HardwareCounter.CacheMisses)}+{nameof(HardwareCounter.InstructionRetired)}"}, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "--counters", $"{nameof(HardwareCounter.CacheMisses)}+{nameof(HardwareCounter.InstructionRetired)}" }, + new OutputLogger(Output)).config; Assert.Equal(2, config.GetHardwareCounters().Count()); - Assert.Single(config.GetHardwareCounters().Where(counter => counter == HardwareCounter.CacheMisses)); - Assert.Single(config.GetHardwareCounters().Where(counter => counter == HardwareCounter.InstructionRetired)); + Assert.Single(config.GetHardwareCounters(), counter => counter == HardwareCounter.CacheMisses); + Assert.Single(config.GetHardwareCounters(), counter => counter == HardwareCounter.InstructionRetired); } [Fact] @@ -482,7 +519,7 @@ public void CanParseDisassemblerWithCustomRecursiveDepth() { const int depth = 123; - var config = ConfigParser.Parse(new[] { "--disasm", "--disasmDepth", depth.ToString()}, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "--disasm", "--disasmDepth", depth.ToString() }, new OutputLogger(Output)).config; var diagnoser = config.GetDiagnosers().OfType().Single(); @@ -518,7 +555,7 @@ public void UserCanSpecifyCustomDefaultJobAndOverwriteItsSettingsViaConsoleArgs( .WithWarmupCount(1) .AsDefault()); - var parsedConfig = ConfigParser.Parse(new[] { "--warmupCount", "2"}, new OutputLogger(Output), globalConfig).config; + var parsedConfig = ConfigParser.Parse(new[] { "--warmupCount", "2" }, new OutputLogger(Output), globalConfig).config; Assert.Equal(2, parsedConfig.GetJobs().Single().Run.WarmupCount); Assert.False(parsedConfig.GetJobs().Single().Meta.IsDefault); // after the merge the job is not "default" anymore @@ -673,4 +710,4 @@ public void CheckUpdateInvalidArgs(string strArgs) Assert.False(isSuccess); } } -} +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs b/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs index 2eae58c7c1..c68134b7cd 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs @@ -249,10 +249,10 @@ public void MissingDependencyIsNotAddedWhenItIsAlreadyPresent() [Fact] public void WhenTwoConfigsAreAddedTheRegularJobsAreJustAdded() { - var configWithClrJob = CreateConfigFromJobs(Job.Default.WithRuntime(CoreRuntime.Core21)); - var configWithCoreJob = CreateConfigFromJobs(Job.Default.WithRuntime(ClrRuntime.Net462)); + var configWithClrJob = CreateConfigFromJobs(Job.Default.WithRuntime(ClrRuntime.Net462)); + var configWithCoreJob = CreateConfigFromJobs(Job.Default.WithRuntime(CoreRuntime.Core80)); - foreach (var added in AddLeftToTheRightAndRightToTheLef(configWithClrJob, configWithCoreJob)) + foreach (var added in AddLeftToTheRightAndRightToTheLef(configWithCoreJob, configWithClrJob)) { var runnableJobs = added.GetJobs(); @@ -269,7 +269,7 @@ public void WhenTwoConfigsAreAddedTheMutatorJobsAreAppliedToAllOtherJobs() var configWithMutatorJob = CreateConfigFromJobs(Job.Default.WithWarmupCount(warmupCount).AsMutator()); var configWithTwoStandardJobs = CreateConfigFromJobs( Job.Default.WithRuntime(ClrRuntime.Net462), - Job.Default.WithRuntime(CoreRuntime.Core21)); + Job.Default.WithRuntime(CoreRuntime.Core80)); foreach (var added in AddLeftToTheRightAndRightToTheLef(configWithTwoStandardJobs, configWithMutatorJob)) { @@ -312,7 +312,7 @@ public void WhenTwoConfigsAreAddedTheMutatorJobsAreAppliedToDefaultJobIfCustomDe Assert.Equal(warmupCount, mergedJob.Run.WarmupCount); Assert.False(mergedJob.Meta.IsDefault); // after the merge the "child" job becomes a standard job Assert.False(mergedJob.Meta.IsMutator); // after the merge the "child" job becomes a standard job - Assert.Single(mergedJob.GetCharacteristicsWithValues(), changedCharacteristic => ReferenceEquals(changedCharacteristic, Jobs.RunMode.WarmupCountCharacteristic)); + Assert.Single(mergedJob.GetCharacteristicsWithValues(), changedCharacteristic => ReferenceEquals(changedCharacteristic, BenchmarkDotNet.Jobs.RunMode.WarmupCountCharacteristic)); } } @@ -376,6 +376,45 @@ public void WhenTwoCustomTimeoutsAreProvidedTheLongerOneIsUsed(bool direction) Assert.Equal(TimeSpan.FromSeconds(2), final.BuildTimeout); } + [Fact] + public void WhenWakeLockIsNotSpecifiedTheDefaultValueIsUsed() + { + var mutable = ManualConfig.CreateEmpty(); + var final = ImmutableConfigBuilder.Create(mutable); + Assert.Equal(DefaultConfig.Instance.WakeLock, final.WakeLock); + } + + [Fact] + public void CustomWakeLockHasPrecedenceOverDefaultWakeLock() + { + WakeLockType customWakeLock = WakeLockType.Display; + var mutable = ManualConfig.CreateEmpty().WithWakeLock(customWakeLock); + var final = ImmutableConfigBuilder.Create(mutable); + Assert.Equal(customWakeLock, final.WakeLock); + } + + [Theory] + [InlineData(WakeLockType.None, WakeLockType.None, WakeLockType.None)] + [InlineData(WakeLockType.System, WakeLockType.None, WakeLockType.None)] + [InlineData(WakeLockType.Display, WakeLockType.None, WakeLockType.Display)] + [InlineData(WakeLockType.None, WakeLockType.System, WakeLockType.None)] + [InlineData(WakeLockType.System, WakeLockType.System, WakeLockType.System)] + [InlineData(WakeLockType.Display, WakeLockType.System, WakeLockType.Display)] + [InlineData(WakeLockType.None, WakeLockType.Display, WakeLockType.Display)] + [InlineData(WakeLockType.System, WakeLockType.Display, WakeLockType.Display)] + [InlineData(WakeLockType.Display, WakeLockType.Display, WakeLockType.Display)] + public void WhenTwoCustomWakeLocksAreProvidedDisplayBeatsNoneBeatsDefault( + WakeLockType left, WakeLockType right, WakeLockType expected) + { + var l = ManualConfig.CreateEmpty().WithWakeLock(left); + var r = ManualConfig.CreateEmpty().WithWakeLock(right); + + l.Add(r); + + var final = ImmutableConfigBuilder.Create(l); + Assert.Equal(expected, final.WakeLock); + } + private static ManualConfig CreateConfigFromJobs(params Job[] jobs) { var config = ManualConfig.CreateEmpty(); diff --git a/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs b/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs index 96f671e60d..d7fb7519f3 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs @@ -60,11 +60,11 @@ public static void Test01Create() Assert.False(j.Environment.Gc.AllowVeryLargeObjects); Assert.Equal(Platform.AnyCpu, j.Environment.Platform); Assert.Equal(RunStrategy.Throughput, j.Run.RunStrategy); // set by default - Assert.Equal("Default", j.Id); // id reset - Assert.True(j.DisplayInfo == "DefaultJob", "DisplayInfo = " + j.DisplayInfo); - Assert.True(j.ResolvedId == "DefaultJob", "ResolvedId = " + j.ResolvedId); + Assert.Equal("CustomId", j.Id); // id remains after unfreeze + Assert.Equal("CustomId", j.DisplayInfo); + Assert.Equal("CustomId", j.ResolvedId); Assert.Equal(j.ResolvedId, j.FolderInfo); - Assert.Equal("Default", j.Environment.Id); + Assert.Equal("CustomId", j.Environment.Id); // id remains after unfreeze // new job j = new Job(j.Freeze()); @@ -160,7 +160,7 @@ public static void Test02Modify() // 4. Freeze-unfreeze: j = j.Freeze().UnfreezeCopy(); - Assert.Equal("Platform=X86, LaunchCount=1", j.Id); + Assert.Equal("SomeId", j.Id); // id not lost Assert.Equal(Platform.X86, j.Environment.Platform); Assert.Equal(1, j.Run.LaunchCount); @@ -204,15 +204,15 @@ public static void Test03IdDoesNotFlow() Assert.False(j.HasValue(CharacteristicObject.IdCharacteristic)); Assert.False(j.Environment.HasValue(CharacteristicObject.IdCharacteristic)); - Job.EnvironmentCharacteristic[j] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); // id will not flow - Assert.False(j.HasValue(CharacteristicObject.IdCharacteristic)); - Assert.False(j.Environment.HasValue(CharacteristicObject.IdCharacteristic)); + Job.EnvironmentCharacteristic[j] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); // id will flow + Assert.True(j.HasValue(CharacteristicObject.IdCharacteristic)); + Assert.True(j.Environment.HasValue(CharacteristicObject.IdCharacteristic)); var c = new CharacteristicSet(EnvironmentMode.LegacyJitX64, RunMode.Long); // id will not flow, new CharacteristicSet Assert.False(c.HasValue(CharacteristicObject.IdCharacteristic)); - Job.EnvironmentCharacteristic[c] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); // id will not flow - Assert.False(c.HasValue(CharacteristicObject.IdCharacteristic)); + Job.EnvironmentCharacteristic[c] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); // id will flow + Assert.True(c.HasValue(CharacteristicObject.IdCharacteristic)); CharacteristicObject.IdCharacteristic[c] = "MyId"; // id set explicitly Assert.Equal("MyId", c.Id); @@ -221,12 +221,12 @@ public static void Test03IdDoesNotFlow() Assert.Equal("MyId", j.Id); Assert.Equal("MyId", j.Environment.Id); - Job.EnvironmentCharacteristic[j] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); // id will not flow - Assert.Equal("MyId", j.Id); - Assert.Equal("MyId", j.Environment.Id); + Job.EnvironmentCharacteristic[j] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); + Assert.Equal("LegacyJitX86", j.Id); + Assert.Equal("LegacyJitX86", j.Environment.Id); - j = j.WithJit(Jit.RyuJit); // custom id will flow - Assert.Equal("MyId", j.Id); + j = j.WithJit(Jit.RyuJit); + Assert.Equal("LegacyJitX86", j.Id); } [Fact] @@ -403,7 +403,7 @@ public static void Test06CharacteristicHacks() [Fact] public static void MutatorAppliedToOtherJobOverwritesOnlyTheConfiguredSettings() { - var jobBefore = Job.Default.WithRuntime(CoreRuntime.Core30); // this is a default job with Runtime set to Core + var jobBefore = Job.Default.WithRuntime(CoreRuntime.Core80); // this is a default job with Runtime set to Core var copy = jobBefore.UnfreezeCopy(); Assert.False(copy.HasValue(RunMode.MaxIterationCountCharacteristic)); @@ -441,6 +441,7 @@ public static void AllJobModesPropertyNamesMatchCharacteristicNames() // it's ma } [Fact] + [Obsolete] public static void WithNuGet() { var j = new Job("SomeId"); @@ -474,6 +475,20 @@ public static void WithNuGet() Assert.Equal(expected, j.Infrastructure.NuGetReferences); // ensure that the list's equality operator returns true when the contents are the same } + [Fact] + public static void UnfreezeCopy_PreservesIdCharacteristic() + { + // Arrange + var original = new Job(); + original.SetValue(Job.IdCharacteristic, "TestID"); + + // Act + var copy = original.UnfreezeCopy(); + + // Assert + Assert.Equal("TestID", copy.GetValue(Job.IdCharacteristic)); + } + private static bool IsSubclassOfobModeOfItself(Type type) { Type jobModeOfT; diff --git a/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs b/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs index 323da8ddba..ef1225432e 100644 --- a/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs +++ b/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs @@ -26,7 +26,7 @@ public class CsProjGeneratorTests [Theory] [InlineData("net471", false)] - [InlineData("netcoreapp3.0", true)] + [InlineData("netcoreapp3.1", true)] public void ItsPossibleToCustomizeProjectSdkBasedOnProjectSdkFromTheProjectFile(string targetFrameworkMoniker, bool isNetCore) { const string withCustomProjectSdk = @" @@ -41,7 +41,7 @@ public void ItsImpossibleToCustomizeProjectSdkForFullFrameworkAppsBasedOnTheImpo { const string withCustomProjectImport = @" - + "; AssertParsedSdkName(withCustomProjectImport, "net471", "Microsoft.NET.Sdk", false); @@ -52,10 +52,10 @@ public void ItsPossibleToCustomizeProjectSdkForNetCoreAppsBasedOnTheImportOfSdk( { const string withCustomProjectImport = @" - + "; - AssertParsedSdkName(withCustomProjectImport, "netcoreapp3.0", "Microsoft.NET.Sdk.WindowsDesktop", true); + AssertParsedSdkName(withCustomProjectImport, "netcoreapp3.1", "Microsoft.NET.Sdk.WindowsDesktop", true); } [AssertionMethod] @@ -87,7 +87,7 @@ public void UseWpfSettingGetsCopied() "; - var sut = new CsProjGenerator("netcoreapp3.0", null, null, null, true); + var sut = new CsProjGenerator("netcoreapp3.1", null, null, null, true); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(withUseWpfTrue); @@ -117,7 +117,7 @@ public void SettingsFromPropsFileImportedUsingAbsolutePathGetCopies() "; - var sut = new CsProjGenerator("netcoreapp3.0", null, null, null, true); + var sut = new CsProjGenerator("netcoreapp3.1", null, null, null, true); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(importingAbsolutePath); @@ -149,7 +149,7 @@ public void SettingsFromPropsFileImportedUsingRelativePathGetCopies() "; - var sut = new CsProjGenerator("netcoreapp3.0", null, null, null, true); + var sut = new CsProjGenerator("netcoreapp3.1", null, null, null, true); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(importingRelativePath); @@ -171,7 +171,7 @@ public void RuntimeHostConfigurationOptionIsCopied() {runtimeHostConfigurationOptionChunk} "; - var sut = new CsProjGenerator("netcoreapp3.0", null, null, null, true); + var sut = new CsProjGenerator("netcoreapp3.1", null, null, null, true); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(source); @@ -186,23 +186,17 @@ public void TheDefaultFilePathShouldBeUsedWhenAnAssemblyLocationIsEmpty() { const string programName = "testProgram"; var config = ManualConfig.CreateEmpty().CreateImmutableConfig(); - var benchmarkMethod = - typeof(MockFactory.MockBenchmarkClass) - .GetTypeInfo() - .GetMethods() - .Single(method => method.Name == nameof(MockFactory.MockBenchmarkClass.Foo)); - //Simulate loading an assembly from a stream var benchmarkDotNetAssembly = typeof(MockFactory.MockBenchmarkClass).GetTypeInfo().Assembly; var streamLoadedAssembly = Assembly.Load(File.ReadAllBytes(benchmarkDotNetAssembly.Location)); - var assemblyType = streamLoadedAssembly.GetRunnableBenchmarks().Select(type => type).FirstOrDefault(); + var assemblyType = streamLoadedAssembly.GetRunnableBenchmarks().Select(type => type).First(); - var target = new Descriptor(assemblyType, benchmarkMethod); + var target = new Descriptor(assemblyType, MockFactory.MockMethodInfo); var benchmarkCase = BenchmarkCase.Create(target, Job.Default, null, config); var benchmarks = new[] { new BenchmarkBuildInfo(benchmarkCase, config.CreateImmutableConfig(), 999) }; - var projectGenerator = new SteamLoadedBuildPartition("netcoreapp3.0", null, null, null, true); + var projectGenerator = new SteamLoadedBuildPartition("netcoreapp3.1", null, null, null, true); string binariesPath = projectGenerator.ResolvePathForBinaries(new BuildPartition(benchmarks, new Resolver()), programName); string expectedPath = Path.Combine(Path.Combine(Directory.GetCurrentDirectory(), "BenchmarkDotNet.Bin"), programName); @@ -213,15 +207,10 @@ public void TheDefaultFilePathShouldBeUsedWhenAnAssemblyLocationIsEmpty() public void TestAssemblyFilePathIsUsedWhenTheAssemblyLocationIsNotEmpty() { const string programName = "testProgram"; - var benchmarkMethod = - typeof(MockFactory.MockBenchmarkClass) - .GetTypeInfo() - .GetMethods() - .Single(method => method.Name == nameof(MockFactory.MockBenchmarkClass.Foo)); - var target = new Descriptor(typeof(MockFactory.MockBenchmarkClass), benchmarkMethod); + var target = new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo); var benchmarkCase = BenchmarkCase.Create(target, Job.Default, null, ManualConfig.CreateEmpty().CreateImmutableConfig()); var benchmarks = new[] { new BenchmarkBuildInfo(benchmarkCase, ManualConfig.CreateEmpty().CreateImmutableConfig(), 0) }; - var projectGenerator = new SteamLoadedBuildPartition("netcoreapp3.0", null, null, null, true); + var projectGenerator = new SteamLoadedBuildPartition("netcoreapp3.1", null, null, null, true); var buildPartition = new BuildPartition(benchmarks, new Resolver()); string binariesPath = projectGenerator.ResolvePathForBinaries(buildPartition, programName); diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/CpuInfoFormatterTests.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/CpuInfoFormatterTests.cs new file mode 100644 index 0000000000..4f487a7b3b --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/CpuInfoFormatterTests.cs @@ -0,0 +1,37 @@ +using System.Text; +using System.Threading.Tasks; +using BenchmarkDotNet.Tests.Infra; +using Perfolizer.Helpers; +using Perfolizer.Models; +using VerifyXunit; +using Xunit; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +[Collection("VerifyTests")] +public class CpuInfoFormatterTests +{ + [Fact] + public Task FormatTest() + { + var captions = new StringBuilder(); + foreach (var processorName in new[] { null, "", "Intel" }) + foreach (var physicalProcessorCount in new int?[] { null, 0, 1, 2 }) + foreach (var physicalCoreCount in new int?[] { null, 0, 1, 2 }) + foreach (var logicalCoreCount in new int?[] { null, 0, 1, 2 }) + { + var cpu = new CpuInfo + { + ProcessorName = processorName, + PhysicalProcessorCount = physicalProcessorCount, + PhysicalCoreCount = physicalCoreCount, + LogicalCoreCount = logicalCoreCount, + }; + + captions.AppendLine(cpu.ToFullBrandName()); + } + + var settings = VerifyHelper.Create(); + return Verifier.Verify(captions.ToString(), settings); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/LinuxCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/LinuxCpuInfoParserTests.cs new file mode 100644 index 0000000000..0d4de2d9cc --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/LinuxCpuInfoParserTests.cs @@ -0,0 +1,173 @@ +using BenchmarkDotNet.Detectors.Cpu.Linux; +using Perfolizer.Models; +using Xunit; +using Xunit.Abstractions; +using static Perfolizer.Horology.Frequency; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +// ReSharper disable StringLiteralTypo +public class LinuxCpuInfoParserTests(ITestOutputHelper output) +{ + private ITestOutputHelper Output { get; } = output; + + [Fact] + public void EmptyTest() + { + var actual = LinuxCpuInfoParser.Parse("", ""); + var expected = new CpuInfo(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void MalformedTest() + { + var actual = LinuxCpuInfoParser.Parse("malformedkey: malformedvalue\n\nmalformedkey2: malformedvalue2", string.Empty); + var expected = new CpuInfo(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void TwoProcessorWithDifferentCoresCountTest() + { + string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoProcessorWithDifferentCoresCount.txt"); + var actual = LinuxCpuInfoParser.Parse(cpuInfo, string.Empty); + var expected = new CpuInfo + { + ProcessorName = "Unknown processor with 2 cores and hyper threading, Unknown processor with 4 cores", + PhysicalProcessorCount = 2, + PhysicalCoreCount = 6, + LogicalCoreCount = 8, + NominalFrequencyHz = 2_500_000_000, + MaxFrequencyHz = 2_500_000_000 + }; + Output.AssertEqual(expected, actual); + } + + + [Fact] + public void RealOneProcessorTwoCoresTest() + { + string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoRealOneProcessorTwoCores.txt"); + var actual = LinuxCpuInfoParser.Parse(cpuInfo, string.Empty); + var expected = new CpuInfo + { + ProcessorName = "Intel(R) Core(TM) i5-6200U CPU @ 2.30GHz", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 2, + LogicalCoreCount = 4, + NominalFrequencyHz = 2_300_000_000, + MaxFrequencyHz = 2_300_000_000 + }; + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealOneProcessorFourCoresTest() + { + string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoRealOneProcessorFourCores.txt"); + var actual = LinuxCpuInfoParser.Parse(cpuInfo, string.Empty); + var expected = new CpuInfo + { + ProcessorName = "Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 4, + LogicalCoreCount = 8, + NominalFrequencyHz = 2_494_300_000, + MaxFrequencyHz = 2_500_000_000 + }; + Output.AssertEqual(expected, actual); + } + + // https://github.com/dotnet/BenchmarkDotNet/issues/2577 + [Fact] + public void Issue2577Test() + { + const string cpuInfo = + """ + processor : 0 + BogoMIPS : 50.00 + Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm lrcpc dcpop asimddp + CPU implementer : 0x41 + CPU architecture: 8 + CPU variant : 0x3 + CPU part : 0xd0c + CPU revision : 1 + """; + const string lscpu = + """ + Architecture: aarch64 + CPU op-mode(s): 32-bit, 64-bit + Byte Order: Little Endian + CPU(s): 16 + On-line CPU(s) list: 0-15 + Vendor ID: ARM + Model name: Neoverse-N1 + Model: 1 + Thread(s) per core: 1 + Core(s) per socket: 16 + Socket(s): 1 + Stepping: r3p1 + BogoMIPS: 50.00 + Flags: fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm lrcpc dcpop asimddp + """; + var actual = LinuxCpuInfoParser.Parse(cpuInfo, lscpu); + var expected = new CpuInfo { ProcessorName = "Neoverse-N1", PhysicalCoreCount = 16 }; + Output.AssertEqual(expected, actual); + } + + [Fact] + public void AmdRyzen9_7950X() + { + string cpuInfo = TestHelper.ReadTestFile("ryzen9-cpuinfo.txt"); + const string lscpu = + """ + Architecture: x86_64 + CPU op-mode(s): 32-bit, 64-bit + Address sizes: 48 bits physical, 48 bits virtual + Byte Order: Little Endian + CPU(s): 32 + On-line CPU(s) list: 0-31 + Vendor ID: AuthenticAMD + Model name: AMD Ryzen 9 7950X 16-Core Processor + CPU family: 25 + Model: 97 + Thread(s) per core: 2 + Core(s) per socket: 16 + Socket(s): 1 + Stepping: 2 + CPU(s) scaling MHz: 41% + CPU max MHz: 5881.0000 + CPU min MHz: 400.0000 + BogoMIPS: 8983.23 + Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl + pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb + bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512 + cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean + flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succo + r smca fsrm flush_l1d + """; + var actual = LinuxCpuInfoParser.Parse(cpuInfo, lscpu); + var expected = new CpuInfo + { + ProcessorName = "AMD Ryzen 9 7950X 16-Core Processor", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 16, + LogicalCoreCount = 32, + NominalFrequencyHz = 400_000_000, + MaxFrequencyHz = 5_881_000_000 + }; + Output.AssertEqual(expected, actual); + } + + [Theory] + [InlineData("Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", 2.50)] + [InlineData("Intel(R) Core(TM) i5-6200U CPU @ 2.30GHz", 2.30)] + [InlineData("Unknown processor with 2 cores and hyper threading, Unknown processor with 4 cores", 0)] + [InlineData("Intel(R) Core(TM) i5-2500 CPU @ 3.30GHz", 3.30)] + public void ParseFrequencyFromBrandStringTests(string brandString, double expectedGHz) + { + var frequency = LinuxCpuInfoParser.ParseFrequencyFromBrandString(brandString) ?? Zero; + Assert.Equal(FromGHz(expectedGHz), frequency); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/PowershellWmiCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/PowershellWmiCpuInfoParserTests.cs new file mode 100644 index 0000000000..a781d70021 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/PowershellWmiCpuInfoParserTests.cs @@ -0,0 +1,122 @@ +using BenchmarkDotNet.Detectors.Cpu.Windows; +using Perfolizer.Models; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +public class PowershellWmiCpuInfoParserTests(ITestOutputHelper output) +{ + private ITestOutputHelper Output { get; } = output; + + + [Fact] + public void EmptyTest() + { + CpuInfo? actual = PowershellWmiCpuInfoParser.Parse(string.Empty); + CpuInfo expected = new CpuInfo(); + Output.AssertEqual(expected, actual); + } + + + [Fact] + public void MalformedTest() + { + CpuInfo? actual = PowershellWmiCpuInfoParser + .Parse("malformedkey=malformedvalue\n\nmalformedkey2=malformedvalue2"); + CpuInfo expected = new CpuInfo(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealTwoProcessorEightCoresTest() + { + const string cpuInfo = + """ + MaxClockSpeed:2400 + Name:Intel(R) Xeon(R) CPU E5-2630 v3 + NumberOfCores:8 + NumberOfLogicalProcessors:16 + + + MaxClockSpeed:2400 + Name:Intel(R) Xeon(R) CPU E5-2630 v3 + NumberOfCores:8 + NumberOfLogicalProcessors:16 + + """; + CpuInfo? actual = PowershellWmiCpuInfoParser.Parse(cpuInfo); + + CpuInfo expected = new CpuInfo + { + ProcessorName = "Intel(R) Xeon(R) CPU E5-2630 v3", + PhysicalProcessorCount = 2, + PhysicalCoreCount = 16, + LogicalCoreCount = 32, + NominalFrequencyHz = 2_400_000_000, + MaxFrequencyHz = 2_400_000_000, + }; + + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealTwoProcessorEightCoresWithWmicBugTest() + { + const string cpuInfo = + "\r\r\n" + + "\r\r\n" + + "MaxClockSpeed:3111\r\r\n" + + "Name:Intel(R) Xeon(R) CPU E5-2687W 0\r\r\n" + + "NumberOfCores:8\r\r\n" + + "NumberOfLogicalProcessors:16\r\r\n" + + "\r\r\n" + + "\r\r\n" + + "MaxClockSpeed:3111\r\r\n" + + "Name:Intel(R) Xeon(R) CPU E5-2687W 0\r\r\n" + + "NumberOfCores:8\r\r\n" + + "NumberOfLogicalProcessors:16\r\r\n" + + "\r\r\n" + + "\r\r\n" + + "\r\r\n"; + + CpuInfo? actual = PowershellWmiCpuInfoParser.Parse(cpuInfo); + + CpuInfo expected = new CpuInfo + { + ProcessorName = "Intel(R) Xeon(R) CPU E5-2687W 0", + PhysicalProcessorCount = 2, + PhysicalCoreCount = 16, + LogicalCoreCount = 32, + NominalFrequencyHz = 3_111_000_000, + MaxFrequencyHz = 3_111_000_000, + }; + + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealOneProcessorFourCoresTest() + { + const string cpuInfo = """ + + MaxClockSpeed:2500 + Name:Intel(R) Core(TM) i7-4710MQ + NumberOfCores:4 + NumberOfLogicalProcessors:8 + """; + + CpuInfo? actual = PowershellWmiCpuInfoParser.Parse(cpuInfo); + CpuInfo expected = new CpuInfo + { + ProcessorName = "Intel(R) Core(TM) i7-4710MQ", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 4, + LogicalCoreCount = 8, + NominalFrequencyHz = 2_500_000_000, + MaxFrequencyHz = 2_500_000_000, + }; + + Output.AssertEqual(expected, actual); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/SysctlCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/SysctlCpuInfoParserTests.cs new file mode 100644 index 0000000000..bdf8cbb10e --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/SysctlCpuInfoParserTests.cs @@ -0,0 +1,45 @@ +using BenchmarkDotNet.Detectors.Cpu.macOS; +using Perfolizer.Models; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +// ReSharper disable StringLiteralTypo +public class SysctlCpuInfoParserTests(ITestOutputHelper output) +{ + private ITestOutputHelper Output { get; } = output; + + [Fact] + public void EmptyTest() + { + var actual = SysctlCpuInfoParser.Parse(string.Empty); + var expected = new CpuInfo(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void MalformedTest() + { + var actual = SysctlCpuInfoParser.Parse("malformedkey=malformedvalue\n\nmalformedkey2=malformedvalue2"); + var expected = new CpuInfo(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealOneProcessorFourCoresTest() + { + string cpuInfo = TestHelper.ReadTestFile("SysctlRealOneProcessorFourCores.txt"); + var actual = SysctlCpuInfoParser.Parse(cpuInfo); + var expected = new CpuInfo + { + ProcessorName = "Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 4, + LogicalCoreCount = 8, + NominalFrequencyHz = 2_200_000_000, + MaxFrequencyHz = 2_200_000_000 + }; + Output.AssertEqual(expected, actual); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoProcessorWithDifferentCoresCount.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoProcessorWithDifferentCoresCount.txt similarity index 100% rename from tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoProcessorWithDifferentCoresCount.txt rename to tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoProcessorWithDifferentCoresCount.txt diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoRealOneProcessorFourCores.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoRealOneProcessorFourCores.txt similarity index 100% rename from tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoRealOneProcessorFourCores.txt rename to tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoRealOneProcessorFourCores.txt diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoRealOneProcessorTwoCores.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoRealOneProcessorTwoCores.txt similarity index 100% rename from tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoRealOneProcessorTwoCores.txt rename to tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoRealOneProcessorTwoCores.txt diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/SysctlRealOneProcessorFourCores.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/SysctlRealOneProcessorFourCores.txt similarity index 100% rename from tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/SysctlRealOneProcessorFourCores.txt rename to tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/SysctlRealOneProcessorFourCores.txt diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ryzen9-cpuinfo.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ryzen9-cpuinfo.txt new file mode 100644 index 0000000000..d942a8d484 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ryzen9-cpuinfo.txt @@ -0,0 +1,896 @@ +processor : 0 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5346.955 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 0 +cpu cores : 16 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 1 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5487.348 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 1 +cpu cores : 16 +apicid : 2 +initial apicid : 2 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 2 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5479.603 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 2 +cpu cores : 16 +apicid : 4 +initial apicid : 4 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 3 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5601.456 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 3 +cpu cores : 16 +apicid : 6 +initial apicid : 6 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 4 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5410.513 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 4 +cpu cores : 16 +apicid : 8 +initial apicid : 8 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 5 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5493.255 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 5 +cpu cores : 16 +apicid : 10 +initial apicid : 10 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 6 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 4422.161 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 6 +cpu cores : 16 +apicid : 12 +initial apicid : 12 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 7 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5505.019 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 7 +cpu cores : 16 +apicid : 14 +initial apicid : 14 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 8 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5414.920 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 8 +cpu cores : 16 +apicid : 16 +initial apicid : 16 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 9 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5060.967 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 9 +cpu cores : 16 +apicid : 18 +initial apicid : 18 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 10 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 3352.555 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 10 +cpu cores : 16 +apicid : 20 +initial apicid : 20 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 11 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5400.772 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 11 +cpu cores : 16 +apicid : 22 +initial apicid : 22 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 12 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5415.333 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 12 +cpu cores : 16 +apicid : 24 +initial apicid : 24 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 13 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 3236.400 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 13 +cpu cores : 16 +apicid : 26 +initial apicid : 26 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 14 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 4519.257 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 14 +cpu cores : 16 +apicid : 28 +initial apicid : 28 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 15 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5414.841 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 15 +cpu cores : 16 +apicid : 30 +initial apicid : 30 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 16 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 4738.471 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 0 +cpu cores : 16 +apicid : 1 +initial apicid : 1 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 17 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5599.000 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 1 +cpu cores : 16 +apicid : 3 +initial apicid : 3 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 18 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5477.067 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 2 +cpu cores : 16 +apicid : 5 +initial apicid : 5 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 19 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5507.543 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 3 +cpu cores : 16 +apicid : 7 +initial apicid : 7 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 20 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 4401.379 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 4 +cpu cores : 16 +apicid : 9 +initial apicid : 9 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 21 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 400.000 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 5 +cpu cores : 16 +apicid : 11 +initial apicid : 11 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 22 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5607.945 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 6 +cpu cores : 16 +apicid : 13 +initial apicid : 13 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 23 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5280.720 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 7 +cpu cores : 16 +apicid : 15 +initial apicid : 15 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 24 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5414.893 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 8 +cpu cores : 16 +apicid : 17 +initial apicid : 17 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 25 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5414.888 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 9 +cpu cores : 16 +apicid : 19 +initial apicid : 19 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 26 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5385.641 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 10 +cpu cores : 16 +apicid : 21 +initial apicid : 21 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 27 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 3130.265 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 11 +cpu cores : 16 +apicid : 23 +initial apicid : 23 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 28 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5400.876 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 12 +cpu cores : 16 +apicid : 25 +initial apicid : 25 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 29 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 4279.794 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 13 +cpu cores : 16 +apicid : 27 +initial apicid : 27 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 30 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 3124.980 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 14 +cpu cores : 16 +apicid : 29 +initial apicid : 29 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 31 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5389.561 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 15 +cpu cores : 16 +apicid : 31 +initial apicid : 31 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestHelper.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestHelper.cs new file mode 100644 index 0000000000..91e6e686a3 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestHelper.cs @@ -0,0 +1,38 @@ +using System.IO; +using System.Reflection; +using JetBrains.Annotations; +using Perfolizer.Helpers; +using Perfolizer.Models; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +public static class TestHelper +{ + public static string ReadTestFile(string name) + { + var assembly = typeof(TestHelper).GetTypeInfo().Assembly; + string resourceName = $"{typeof(TestHelper).Namespace}.TestFiles.{name}"; + + using var stream = assembly.GetManifestResourceStream(resourceName); + if (stream == null) + throw new FileNotFoundException($"Resource {resourceName} not found in {assembly.FullName}"); + + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } + + [AssertionMethod] + public static void AssertEqual(this ITestOutputHelper output, CpuInfo expected, CpuInfo actual) + { + output.WriteLine($"Expected : {expected.ToFullBrandName()}"); + output.WriteLine($"Actual : {actual.ToFullBrandName()}"); + Assert.Equal(expected.ProcessorName, actual.ProcessorName); + Assert.Equal(expected.PhysicalProcessorCount, actual.PhysicalProcessorCount); + Assert.Equal(expected.PhysicalCoreCount, actual.PhysicalCoreCount); + Assert.Equal(expected.LogicalCoreCount, actual.LogicalCoreCount); + Assert.Equal(expected.NominalFrequency(), actual.NominalFrequency()); + Assert.Equal(expected.MaxFrequency(), actual.MaxFrequency()); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/VerifiedFiles/CpuInfoFormatterTests.FormatTest.verified.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/VerifiedFiles/CpuInfoFormatterTests.FormatTest.verified.txt similarity index 100% rename from tests/BenchmarkDotNet.Tests/Portability/Cpu/VerifiedFiles/CpuInfoFormatterTests.FormatTest.verified.txt rename to tests/BenchmarkDotNet.Tests/Detectors/Cpu/VerifiedFiles/CpuInfoFormatterTests.FormatTest.verified.txt diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/WmicCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/WmicCpuInfoParserTests.cs new file mode 100644 index 0000000000..8903f98360 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/WmicCpuInfoParserTests.cs @@ -0,0 +1,115 @@ +using BenchmarkDotNet.Detectors.Cpu.Windows; +using Perfolizer.Models; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +// ReSharper disable StringLiteralTypo +public class WmicCpuInfoParserTests(ITestOutputHelper output) +{ + private ITestOutputHelper Output { get; } = output; + + [Fact] + public void EmptyTest() + { + var actual = WmicCpuInfoParser.Parse(string.Empty); + var expected = new CpuInfo(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void MalformedTest() + { + var actual = WmicCpuInfoParser.Parse("malformedkey=malformedvalue\n\nmalformedkey2=malformedvalue2"); + var expected = new CpuInfo(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealTwoProcessorEightCoresTest() + { + const string cpuInfo = @" + +MaxClockSpeed=2400 +Name=Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz +NumberOfCores=8 +NumberOfLogicalProcessors=16 + + +MaxClockSpeed=2400 +Name=Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz +NumberOfCores=8 +NumberOfLogicalProcessors=16 + +"; + var actual = WmicCpuInfoParser.Parse(cpuInfo); + var expected = new CpuInfo + { + ProcessorName = "Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz", + PhysicalProcessorCount = 2, + PhysicalCoreCount = 16, + LogicalCoreCount = 32, + NominalFrequencyHz = 2_400_000_000, + MaxFrequencyHz = 2_400_000_000, + }; + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealTwoProcessorEightCoresWithWmicBugTest() + { + const string cpuInfo = + "\r\r\n" + + "\r\r\n" + + "MaxClockSpeed=3111\r\r\n" + + "Name=Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz\r\r\n" + + "NumberOfCores=8\r\r\n" + + "NumberOfLogicalProcessors=16\r\r\n" + + "\r\r\n" + + "\r\r\n" + + "MaxClockSpeed=3111\r\r\n" + + "Name=Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz\r\r\n" + + "NumberOfCores=8\r\r\n" + + "NumberOfLogicalProcessors=16\r\r\n" + + "\r\r\n" + + "\r\r\n" + + "\r\r\n"; + var actual = WmicCpuInfoParser.Parse(cpuInfo); + var expected = new CpuInfo + { + ProcessorName = "Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz", + PhysicalProcessorCount = 2, + PhysicalCoreCount = 16, + LogicalCoreCount = 32, + NominalFrequencyHz = 3_111_000_000, + MaxFrequencyHz = 3_111_000_000, + }; + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealOneProcessorFourCoresTest() + { + const string cpuInfo = @" + +MaxClockSpeed=2500 +Name=Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz +NumberOfCores=4 +NumberOfLogicalProcessors=8 + +"; + + var actual = WmicCpuInfoParser.Parse(cpuInfo); + var expected = new CpuInfo + { + ProcessorName = "Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 4, + LogicalCoreCount = 8, + NominalFrequencyHz = 2_500_000_000, + MaxFrequencyHz = 2_500_000_000, + }; + Output.AssertEqual(expected, actual); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineGeneralStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineActualStageTests.cs similarity index 85% rename from tests/BenchmarkDotNet.Tests/Engine/EngineGeneralStageTests.cs rename to tests/BenchmarkDotNet.Tests/Engine/EngineActualStageTests.cs index 5811a0f68c..97078eb953 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineGeneralStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineActualStageTests.cs @@ -36,17 +36,14 @@ private void AutoTest(Func measure, int min, int ma if (max == -1) max = min; var job = Job.Default; - var stage = CreateStage(job, measure); - var measurements = stage.Run(1, iterationMode, true, 1); + var engine = new MockEngine(output, job, measure); + var stage = iterationMode == IterationMode.Overhead + ? EngineActualStage.GetOverhead(engine) + : EngineActualStage.GetWorkload(engine, RunStrategy.Throughput); + var (_, measurements) = engine.Run(stage); int count = measurements.Count; output.WriteLine($"MeasurementCount = {count} (Min= {min}, Max = {max})"); Assert.InRange(count, min, max); } - - private EngineActualStage CreateStage(Job job, Func measure) - { - var engine = new MockEngine(output, job, measure); - return new EngineActualStage(engine); - } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs index d5b7b8c65c..fd3f4887ff 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; using BenchmarkDotNet.Tests.Mocks; using Perfolizer.Horology; using Xunit; @@ -46,8 +48,8 @@ private void AutoTest(Frequency clockFrequency, TimeInterval operationTime, doub Infrastructure = { Clock = new MockClock(clockFrequency) }, Accuracy = { MaxRelativeError = maxRelativeError } }.Freeze(); - var stage = CreateStage(job, data => data.InvokeCount * operationTime); - long invokeCount = stage.Run().PerfectInvocationCount; + var engine = new MockEngine(output, job, data => data.InvokeCount * operationTime); + var (invokeCount, _) = engine.Run(EnginePilotStage.GetStage(engine)); output.WriteLine($"InvokeCount = {invokeCount} (Min= {minInvokeCount}, Max = {MaxPossibleInvokeCount})"); Assert.InRange(invokeCount, minInvokeCount, MaxPossibleInvokeCount); } @@ -59,16 +61,10 @@ private void SpecificTest(TimeInterval iterationTime, TimeInterval operationTime Infrastructure = { Clock = new MockClock(Frequency.MHz) }, Run = { IterationTime = iterationTime } }.Freeze(); - var stage = CreateStage(job, data => data.InvokeCount * operationTime); - long invokeCount = stage.Run().PerfectInvocationCount; + var engine = new MockEngine(output, job, data => data.InvokeCount * operationTime); + var (invokeCount, _) = engine.Run(EnginePilotStage.GetStage(engine)); output.WriteLine($"InvokeCount = {invokeCount} (Min= {minInvokeCount}, Max = {maxInvokeCount})"); Assert.InRange(invokeCount, minInvokeCount, maxInvokeCount); } - - private EnginePilotStage CreateStage(Job job, Func measure) - { - var engine = new MockEngine(output, job, measure); - return new EnginePilotStage(engine); - } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs index 1f966fbcaa..b9c465b87b 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs @@ -16,7 +16,7 @@ public class EngineWarmupStageTests { private const int MinIterationCount = EngineResolver.DefaultMinWarmupIterationCount; private const int MaxIterationCount = EngineResolver.DefaultMaxWarmupIterationCount; - private const int MaxOverheadIterationCount = DefaultStoppingCriteriaFactory.MaxOverheadIterationCount; + private const int MaxOverheadIterationCount = EngineWarmupStage.MaxOverheadIterationCount; private readonly ITestOutputHelper output; @@ -85,17 +85,14 @@ private void AutoTest(Func measure, int min, int ma { if (max == -1) max = min; - var stage = CreateStage(job ?? Job.Default, measure); - var measurements = stage.Run(1, mode, 1, RunStrategy.Throughput); + var engine = new MockEngine(output, job ?? Job.Default, measure); + var stage = mode == IterationMode.Overhead + ? EngineWarmupStage.GetOverhead() + : EngineWarmupStage.GetWorkload(engine, RunStrategy.Throughput); + var (_, measurements) = engine.Run(stage); int count = measurements.Count; output.WriteLine($"MeasurementCount = {count} (Min= {min}, Max = {max})"); Assert.InRange(count, min, max); } - - private EngineWarmupStage CreateStage(Job job, Func measure) - { - var engine = new MockEngine(output, job, measure); - return new EngineWarmupStage(engine); - } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/AutoWarmupStoppingCriteriaTests.cs b/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/AutoWarmupStoppingCriteriaTests.cs deleted file mode 100644 index 39eb94a225..0000000000 --- a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/AutoWarmupStoppingCriteriaTests.cs +++ /dev/null @@ -1,80 +0,0 @@ -using System.Collections.Generic; -using BenchmarkDotNet.Engines; -using JetBrains.Annotations; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests.Engine.StoppingCriteria -{ - public class AutoWarmupStoppingCriteriaTests : StoppingCriteriaTestsBase - { - public AutoWarmupStoppingCriteriaTests(ITestOutputHelper output) : base(output) { } - - [Theory] - [MemberData(nameof(EvaluateData))] - public void EvaluateTest(int minIterationCount, int maxIterationCount, int minFluctuationCount, double[] values, int expectedCount) - { - var criteria = new AutoWarmupStoppingCriteria(minIterationCount, maxIterationCount, minFluctuationCount); - ResolutionsAreCorrect(criteria, values, expectedCount); - } - - public static IEnumerable EvaluateData() - { - yield return new object[] { 0, 8, 4, new double[] { 0, 10, 0, 10, 0, 10, 0, 10, 0, 10 }, 5 }; - yield return new object[] { 0, 5, 1, new double[] { 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 }, 5 }; - yield return new object[] { 0, 5, 1, new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 2 }; - yield return new object[] { 0, 600, 2, new double[] { 10, 9, 8, 7, 8, 9, 8, 7, 8, 9 }, 7 }; - yield return new object[] { 5, 8, 0, new double[] { 0, 10, 0, 10, 0, 10, 0, 10, 0, 10 }, 5 }; - } - - [Theory] - [InlineData(0, 0, 0, "AutoWarmupStoppingCriteria(minIterationCount=0, maxIterationCount=0, minFluctuationCount=0)")] - [InlineData(1, 22, 333, "AutoWarmupStoppingCriteria(minIterationCount=1, maxIterationCount=22, minFluctuationCount=333)")] - public void AutoWarmupTitleTest(int minIterationCount, int maxIterationCount, int minFluctuationCount, string expectedTitle) - { - var criteria = new AutoWarmupStoppingCriteria(minIterationCount, maxIterationCount, minFluctuationCount); - Assert.Equal(expectedTitle, criteria.Title); - } - - [Theory] - [MemberData(nameof(WarningsDataNames))] - public void WarningsTest(string dataName) - { - var (criteria, expectedWarnings) = WarningsData[dataName]; - Output.WriteLine("Criteria:" + criteria.Title); - Assert.Equal(expectedWarnings.Messages, criteria.Warnings); - } - - private static readonly IDictionary WarningsData = new Dictionary - { - { - "0/0/0", new WaringTestData( - new AutoWarmupStoppingCriteria(0, 0, 0), - Warnings.Empty) - }, - { - "5/4/0", new WaringTestData( - new AutoWarmupStoppingCriteria(5, 4, 0), - new Warnings("Min Iteration Count (0) is greater than Max Iteration Count (4)")) - }, - { - "-5/0/0", new WaringTestData( - new AutoWarmupStoppingCriteria(-5, 0, 0), - new Warnings("Min Iteration Count (-5) is negative")) - }, - { - "0/0/-5", new WaringTestData( - new AutoWarmupStoppingCriteria(0, 0, -5), - new Warnings("Min Fluctuation Count (-5) is negative")) - }, - { - "0/-5/0", new WaringTestData( - new AutoWarmupStoppingCriteria(0, -5, 0), - new Warnings("Max Iteration Count (-5) is negative", "Min Iteration Count (0) is greater than Max Iteration Count (-5)")) - } - }; - - [UsedImplicitly] - public static TheoryData WarningsDataNames = TheoryDataHelper.Create(WarningsData.Keys); - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/FixedStoppingCriteriaTests.cs b/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/FixedStoppingCriteriaTests.cs deleted file mode 100644 index 0a352d8c74..0000000000 --- a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/FixedStoppingCriteriaTests.cs +++ /dev/null @@ -1,67 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using BenchmarkDotNet.Engines; -using JetBrains.Annotations; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests.Engine.StoppingCriteria -{ - public class FixedStoppingCriteriaTests : StoppingCriteriaTestsBase - { - public FixedStoppingCriteriaTests(ITestOutputHelper output) : base(output) { } - - [Theory] - [InlineData(1)] - [InlineData(2)] - [InlineData(10)] - [InlineData(100)] - public void EvaluateTest(int expectedCount) - { - var values = Enumerable.Range(0, expectedCount * 2).Select(x => (double) x).ToArray(); - var criteria = new FixedStoppingCriteria(expectedCount); - ResolutionsAreCorrect(criteria, values, expectedCount); - } - - [Theory] - [InlineData(0, "FixedStoppingCriteria(iterationCount=0)")] - [InlineData(1, "FixedStoppingCriteria(iterationCount=1)")] - [InlineData(42, "FixedStoppingCriteria(iterationCount=42)")] - public void TitleTest(int iterationTitle, string expectedTitle) - { - var criteria = new FixedStoppingCriteria(iterationTitle); - Assert.Equal(expectedTitle, criteria.Title); - } - - [Theory] - [MemberData(nameof(WarningsDataNames))] - public void WarningsTest(string dataName) - { - var (criteria, expectedWarnings) = WarningsData[dataName]; - Output.WriteLine("Criteria:" + criteria.Title); - Assert.Equal(expectedWarnings.Messages, criteria.Warnings); - } - - private static readonly IDictionary WarningsData = new Dictionary - { - { - "0", new WaringTestData( - new FixedStoppingCriteria(0), - Warnings.Empty) - }, - { - "1", new WaringTestData( - new FixedStoppingCriteria(1), - Warnings.Empty) - }, - { - "-1", new WaringTestData( - new FixedStoppingCriteria(-1), - new Warnings("Iteration count (-1) is negative")) - } - }; - - [UsedImplicitly] - public static TheoryData WarningsDataNames = TheoryDataHelper.Create(WarningsData.Keys); - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/StoppingCriteriaTestsBase.cs b/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/StoppingCriteriaTestsBase.cs deleted file mode 100644 index 39b83da67a..0000000000 --- a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/StoppingCriteriaTestsBase.cs +++ /dev/null @@ -1,73 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Reports; -using JetBrains.Annotations; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests.Engine.StoppingCriteria -{ - [Collection("StoppingCriteriaTests")] - public abstract class StoppingCriteriaTestsBase - { - protected readonly ITestOutputHelper Output; - - protected StoppingCriteriaTestsBase(ITestOutputHelper output) => Output = output; - - [AssertionMethod] - protected void ResolutionsAreCorrect(IStoppingCriteria criteria, double[] values, int expectedCount) - { - var measurements = Generate(values); - for (int iteration = 1; iteration <= measurements.Count; iteration++) - { - var subMeasurements = measurements.Take(iteration).ToList(); - var resolution = criteria.Evaluate(subMeasurements); - bool actualIsFinished = resolution.IsFinished; - bool expectedIsFinished = iteration >= expectedCount; - Output.WriteLine($"#{iteration} " + - $"Expected: {expectedIsFinished}, " + - $"Actual: {actualIsFinished}{(resolution.Message != null ? $" [{resolution.Message}]" : "")}, " + - $"Measurements: <{string.Join(",", values.Take(iteration))}>"); - Assert.Equal(expectedIsFinished, actualIsFinished); - } - } - - private static IReadOnlyList Generate(params double[] values) - { - var measurements = new List(values.Length); - for (int i = 1; i <= values.Length; i++) - measurements.Add(new Measurement(1, IterationMode.Unknown, IterationStage.Unknown, i, 1, values[i - 1])); - return measurements; - } - - public class Warnings - { - public static readonly Warnings Empty = new Warnings(); - - public string[] Messages { get; } - - public Warnings(params string[] messages) => Messages = messages; - - public override string ToString() => $"Messages: {Messages.Length}"; - } - - public class WaringTestData - { - private IStoppingCriteria Criteria { get; } - private Warnings ExpectedWarnings { get; } - - public WaringTestData(IStoppingCriteria criteria, Warnings expectedWarnings) - { - Criteria = criteria; - ExpectedWarnings = expectedWarnings; - } - - public void Deconstruct(out IStoppingCriteria criteria, out Warnings expectedWarnings) - { - criteria = Criteria; - expectedWarnings = ExpectedWarnings; - } - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Environments/HostEnvironmentInfoTests.cs b/tests/BenchmarkDotNet.Tests/Environments/HostEnvironmentInfoTests.cs index 977be28869..a49d9ea2e7 100644 --- a/tests/BenchmarkDotNet.Tests/Environments/HostEnvironmentInfoTests.cs +++ b/tests/BenchmarkDotNet.Tests/Environments/HostEnvironmentInfoTests.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Portability; using BenchmarkDotNet.Tests.Builders; using JetBrains.Annotations; +using Perfolizer.Helpers; using Xunit; namespace BenchmarkDotNet.Tests.Environments @@ -22,7 +23,7 @@ public void ReturnsHypervisorNameWhenItsDetected(string hypervisorName) string line = info.ToFormattedString().First(); string expected = $"{HostEnvironmentInfo.BenchmarkDotNetCaption} v{info.BenchmarkDotNetVersion}, " + - $"{info.OsVersion.Value} ({hypervisor.Name})"; + $"{info.Os.Value.ToBrandString()} ({hypervisor.Name})"; Assert.Equal(expected, line); } @@ -46,7 +47,7 @@ public void DoesntReturnHypervisorNameWhenItsNotDetected() string line = info.ToFormattedString().First(); string expected = $"{HostEnvironmentInfo.BenchmarkDotNetCaption} v{info.BenchmarkDotNetVersion}, " + - $"{info.OsVersion.Value}"; + $"{info.Os.Value.ToBrandString()}"; Assert.Equal(expected, line); } } diff --git a/tests/BenchmarkDotNet.Tests/Environments/OsBrandStringTests.cs b/tests/BenchmarkDotNet.Tests/Environments/OsBrandStringTests.cs deleted file mode 100644 index 5690fbe6b7..0000000000 --- a/tests/BenchmarkDotNet.Tests/Environments/OsBrandStringTests.cs +++ /dev/null @@ -1,70 +0,0 @@ -using BenchmarkDotNet.Environments; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests.Environments -{ - public class OsBrandStringTests - { - private readonly ITestOutputHelper output; - - public OsBrandStringTests(ITestOutputHelper output) => this.output = output; - - private void Check(string actual, string expected) - { - output.WriteLine("LENGTH : " + actual.Length); - output.WriteLine("ACTUAL : " + actual); - output.WriteLine("EXPECTED : " + expected); - Assert.Equal(expected, actual); - - // The line with OsBrandString is one of the longest lines in the summary. - // When people past in on GitHub, it can be a reason of an ugly horizontal scrollbar. - // To avoid this, we are trying to minimize this line and use the minimum possible number of characters. - // In this test, we check that the length of the OS brand string for typical Windows versions - // is less than 61 characters. - const int maxLength = 61; - Assert.True(actual.Length <= maxLength, $"The brand string name length should be <= {maxLength}"); - } - - [Theory] - [InlineData("6.3.9600", "Windows 8.1 (6.3.9600)")] - [InlineData("10.0.14393", "Windows 10 (10.0.14393/1607/AnniversaryUpdate/Redstone1)")] - public void WindowsIsPrettified(string originalVersion, string prettifiedName) - => Check(OsBrandStringHelper.Prettify("Windows", originalVersion), prettifiedName); - - [Theory] - [InlineData("10.0.10240", 17797, "Windows 10 (10.0.10240.17797/1507/RTM/Threshold1)")] - [InlineData("10.0.10586", 1478, "Windows 10 (10.0.10586.1478/1511/NovemberUpdate/Threshold2)")] - [InlineData("10.0.14393", 2156, "Windows 10 (10.0.14393.2156/1607/AnniversaryUpdate/Redstone1)")] - [InlineData("10.0.15063", 997, "Windows 10 (10.0.15063.997/1703/CreatorsUpdate/Redstone2)")] - [InlineData("10.0.16299", 334, "Windows 10 (10.0.16299.334/1709/FallCreatorsUpdate/Redstone3)")] - [InlineData("10.0.17134", 48, "Windows 10 (10.0.17134.48/1803/April2018Update/Redstone4)")] - [InlineData("10.0.17763", 1, "Windows 10 (10.0.17763.1/1809/October2018Update/Redstone5)")] - [InlineData("10.0.18362", 693, "Windows 10 (10.0.18362.693/1903/May2019Update/19H1)")] - [InlineData("10.0.18363", 657, "Windows 10 (10.0.18363.657/1909/November2019Update/19H2)")] - [InlineData("10.0.19041", 1, "Windows 10 (10.0.19041.1/2004/May2020Update/20H1)")] - [InlineData("10.0.19042", 746, "Windows 10 (10.0.19042.746/20H2/October2020Update)")] - [InlineData("10.0.19043", 964, "Windows 10 (10.0.19043.964/21H1/May2021Update)")] - [InlineData("10.0.19044", 1147, "Windows 10 (10.0.19044.1147/21H2/November2021Update)")] - [InlineData("10.0.19099", 1729, "Windows 10 (10.0.19099.1729)")] - [InlineData("10.0.19045", 0, "Windows 10 (10.0.19045.0/22H2/2022Update)")] - [InlineData("10.0.22000", 348, "Windows 11 (10.0.22000.348/21H2/SunValley)")] - [InlineData("10.0.22518", 1012, "Windows 11 (10.0.22518.1012)")] - [InlineData("10.0.22621", 0, "Windows 11 (10.0.22621.0/22H2/2022Update/SunValley2)")] - [InlineData("10.0.22631", 2428, "Windows 11 (10.0.22631.2428/23H2/2023Update/SunValley3)")] - public void WindowsWithUbrIsPrettified(string originalVersion, int ubr, string prettifiedName) - => Check(OsBrandStringHelper.Prettify("Windows", originalVersion, ubr), prettifiedName); - - [Theory] - [InlineData("macOS 10.12.6 (16G29)", "Darwin 16.7.0", "macOS Sierra 10.12.6 (16G29) [Darwin 16.7.0]")] - [InlineData("macOS 10.13.4 (17E199)", "Darwin 17.5.0", "macOS High Sierra 10.13.4 (17E199) [Darwin 17.5.0]")] - [InlineData("macOS 10.15.4 (19E266)", "Darwin 19.4.0", "macOS Catalina 10.15.4 (19E266) [Darwin 19.4.0]")] - [InlineData("macOS 11.1 (20C69)", "Darwin 20.2.0", "macOS Big Sur 11.1 (20C69) [Darwin 20.2.0]")] - [InlineData("macOS 11.3.1 (20E241)", "Darwin 20.4.0", "macOS Big Sur 11.3.1 (20E241) [Darwin 20.4.0]")] - [InlineData("macOS 12.1 (21C52)", "Darwin 21.2.0", "macOS Monterey 12.1 (21C52) [Darwin 21.2.0]")] - [InlineData("macOS 13.0.1 (22A400)", "Darwin 22.1.0", "macOS Ventura 13.0.1 (22A400) [Darwin 22.1.0]")] - [InlineData("macOS 14.0.0", "Darwin 23.0.0", "macOS Sonoma 14.0.0 [Darwin 23.0.0]")] - public void MacOSXIsPrettified(string systemVersion, string kernelVersion, string prettifiedName) - => Check(OsBrandStringHelper.PrettifyMacOSX(systemVersion, kernelVersion), prettifiedName); - } -} diff --git a/tests/BenchmarkDotNet.Tests/Environments/ProcessorBrandStringTests.cs b/tests/BenchmarkDotNet.Tests/Environments/ProcessorBrandStringTests.cs deleted file mode 100644 index 7911d5a135..0000000000 --- a/tests/BenchmarkDotNet.Tests/Environments/ProcessorBrandStringTests.cs +++ /dev/null @@ -1,73 +0,0 @@ -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Portability.Cpu; -using Perfolizer.Horology; -using Xunit; - -namespace BenchmarkDotNet.Tests.Environments -{ - public class ProcessorBrandStringTests - { - [Theory] - [InlineData("Intel(R) Pentium(TM) G4560 CPU @ 3.50GHz", "Intel Pentium G4560 CPU 3.50GHz")] - [InlineData("Intel(R) Core(TM) i7 CPU 970 @ 3.20GHz", "Intel Core i7 CPU 970 3.20GHz (Nehalem)")] // Nehalem/Westmere/Gulftown - [InlineData("Intel(R) Core(TM) i7-920 CPU @ 2.67GHz", "Intel Core i7-920 CPU 2.67GHz (Nehalem)")] - [InlineData("Intel(R) Core(TM) i7-2600 CPU @ 3.40GHz", "Intel Core i7-2600 CPU 3.40GHz (Sandy Bridge)")] - [InlineData("Intel(R) Core(TM) i7-3770 CPU @ 3.40GHz", "Intel Core i7-3770 CPU 3.40GHz (Ivy Bridge)")] - [InlineData("Intel(R) Core(TM) i7-4770K CPU @ 3.50GHz", "Intel Core i7-4770K CPU 3.50GHz (Haswell)")] - [InlineData("Intel(R) Core(TM) i7-5775R CPU @ 3.30GHz", "Intel Core i7-5775R CPU 3.30GHz (Broadwell)")] - [InlineData("Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz", "Intel Core i7-6700HQ CPU 2.60GHz (Skylake)")] - [InlineData("Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz", "Intel Core i7-7700 CPU 3.60GHz (Kaby Lake)")] - [InlineData("Intel(R) Core(TM) i7-8650U CPU @ 1.90GHz ", "Intel Core i7-8650U CPU 1.90GHz (Kaby Lake R)")] - [InlineData("Intel(R) Core(TM) i7-8700K CPU @ 3.70GHz", "Intel Core i7-8700K CPU 3.70GHz (Coffee Lake)")] - public void IntelCoreIsPrettified(string originalName, string prettifiedName) => - Assert.Equal(prettifiedName, ProcessorBrandStringHelper.Prettify(new CpuInfo(originalName, nominalFrequency: null))); - - [Theory] - [InlineData("Intel(R) Pentium(TM) G4560 CPU @ 3.50GHz", "Intel Pentium G4560 CPU 3.50GHz (Max: 3.70GHz)", 3.7)] - [InlineData("Intel(R) Core(TM) i5-2500 CPU @ 3.30GHz", "Intel Core i5-2500 CPU 3.30GHz (Max: 3.70GHz) (Sandy Bridge)", 3.7)] - [InlineData("Intel(R) Core(TM) i7-2600 CPU @ 3.40GHz", "Intel Core i7-2600 CPU 3.40GHz (Max: 3.70GHz) (Sandy Bridge)", 3.7)] - [InlineData("Intel(R) Core(TM) i7-3770 CPU @ 3.40GHz", "Intel Core i7-3770 CPU 3.40GHz (Max: 3.50GHz) (Ivy Bridge)", 3.5)] - [InlineData("Intel(R) Core(TM) i7-4770K CPU @ 3.50GHz", "Intel Core i7-4770K CPU 3.50GHz (Max: 3.60GHz) (Haswell)", 3.6)] - [InlineData("Intel(R) Core(TM) i7-5775R CPU @ 3.30GHz", "Intel Core i7-5775R CPU 3.30GHz (Max: 3.40GHz) (Broadwell)", 3.4)] - public void CoreIsPrettifiedWithDiffFrequencies(string originalName, string prettifiedName, double actualFrequency) - { - Assert.Equal(prettifiedName, ProcessorBrandStringHelper.Prettify(new CpuInfo(originalName, nominalFrequency: Frequency.FromGHz(actualFrequency)), includeMaxFrequency: true)); - } - - [Theory] - [InlineData("AMD Ryzen 7 2700X Eight-Core Processor", "AMD Ryzen 7 2700X 4.10GHz", 4.1, 8, 16)] - [InlineData("AMD Ryzen 7 2700X Eight-Core Processor", "AMD Ryzen 7 2700X Eight-Core Processor 4.10GHz", 4.1, null, null)] - public void AmdIsPrettifiedWithDiffFrequencies(string originalName, string prettifiedName, double actualFrequency, int? physicalCoreCount, int? logicalCoreCount) - { - var cpuInfo = new CpuInfo( - originalName, - physicalProcessorCount: null, - physicalCoreCount, - logicalCoreCount, - Frequency.FromGHz(actualFrequency), - minFrequency: null, - maxFrequency: null); - - Assert.Equal(prettifiedName, ProcessorBrandStringHelper.Prettify(cpuInfo, includeMaxFrequency: true)); - } - - [Theory] - [InlineData("8130U", "Kaby Lake")] - [InlineData("9900K", "Coffee Lake")] - [InlineData("8809G", "Kaby Lake G")] - public void IntelCoreMicroarchitecture(string modelNumber, string microarchitecture) - { - Assert.Equal(microarchitecture, ProcessorBrandStringHelper.ParseIntelCoreMicroarchitecture(modelNumber)); - } - - [Theory] - [InlineData("", "Unknown processor")] - [InlineData(null, "Unknown processor")] - public void UnknownProcessorDoesNotThrow(string originalName, string prettifiedName) - { - var cpuInfo = new CpuInfo(originalName, nominalFrequency: null); - - Assert.Equal(prettifiedName, ProcessorBrandStringHelper.Prettify(cpuInfo, includeMaxFrequency: true)); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs b/tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs index b3e142c6c5..7799fd2038 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs +++ b/tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs @@ -5,12 +5,16 @@ using System.Threading.Tasks; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Exporters.Json; using BenchmarkDotNet.Exporters.Xml; using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; using BenchmarkDotNet.Tests.Builders; +using BenchmarkDotNet.Tests.Infra; using BenchmarkDotNet.Tests.Mocks; +using BenchmarkDotNet.Tests.Reports; using JetBrains.Annotations; using VerifyXunit; using Xunit; @@ -18,7 +22,6 @@ namespace BenchmarkDotNet.Tests.Exporters { [Collection("VerifyTests")] - [UsesVerify] public class CommonExporterVerifyTests : IDisposable { private readonly CultureInfo initCulture; @@ -51,10 +54,15 @@ public Task Exporters(string cultureInfoName) foreach (var exporter in exporters) { PrintTitle(logger, exporter); - exporter.ExportToLog(MockFactory.CreateSummary(config.WithCultureInfo(cultureInfo)), logger); + exporter.ExportToLog( + MockFactory.CreateSummary( + config.WithCultureInfo(cultureInfo), + hugeSd: false, + [new Metric(new FakeMetricDescriptor("CacheMisses", "Hardware counter 'CacheMisses' per single operation", "N0"), 7)]), + logger); } - var settings = VerifySettingsFactory.Create(); + var settings = VerifyHelper.Create(); settings.UseTextForParameters(GetName(cultureInfo)); return Verifier.Verify(logger.GetLog(), settings); } @@ -98,6 +106,8 @@ private static IEnumerable GetExporters() .AddColumn(StatisticColumn.Mean) .AddColumn(StatisticColumn.StdDev) .AddColumn(StatisticColumn.P67) + .AddHardwareCounters(HardwareCounter.CacheMisses) + .AddColumnProvider(DefaultColumnProviders.Metrics) .AddDiagnoser(Diagnosers.MemoryDiagnoser.Default); public void Dispose() diff --git a/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterVerifyTests.cs b/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterVerifyTests.cs index 950f8498f8..2d57e308a0 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterVerifyTests.cs +++ b/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterVerifyTests.cs @@ -10,6 +10,7 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Tests.Builders; +using BenchmarkDotNet.Tests.Infra; using BenchmarkDotNet.Validators; using JetBrains.Annotations; using VerifyXunit; @@ -18,12 +19,9 @@ namespace BenchmarkDotNet.Tests.Exporters { [Collection("VerifyTests")] - [UsesVerify] public class MarkdownExporterVerifyTests : IDisposable { - private readonly CultureInfo initCulture; - - public MarkdownExporterVerifyTests() => initCulture = Thread.CurrentThread.CurrentCulture; + private readonly CultureInfo initCulture = Thread.CurrentThread.CurrentCulture; [UsedImplicitly] public static TheoryData GetGroupBenchmarkTypes() @@ -54,7 +52,7 @@ public Task GroupExporterTest(Type benchmarkType) foreach (var error in errors) logger.WriteLineError("* " + error.Message); - var settings = VerifySettingsFactory.Create(); + var settings = VerifyHelper.Create(); settings.UseTextForParameters(benchmarkType.Name); return Verifier.Verify(logger.GetLog(), settings); } diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_Invariant.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_Invariant.verified.txt index cd3482c04f..b406e20dd1 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_Invariant.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_Invariant.verified.txt @@ -4,18 +4,18 @@ AsciiDocExporter .... BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 .... [options="header"] |=== -|Method |Mean |Error |StdDev |P67 -|Foo | 1.000 ns| NA| 0.000 ns| 1.000 ns -|Bar | 1.000 ns| NA| 0.000 ns| 1.000 ns +|Method |Mean |Error |StdDev |P67 |CacheMisses +|Foo | 1.000 ns| 0.000 ns| 0.000 ns| 1.000 ns| 7 +|Bar | 1.000 ns| 0.000 ns| 0.000 ns| 1.000 ns| 7 |=== ############################################ HtmlExporter @@ -37,19 +37,19 @@ HtmlExporter 

 BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V)
 MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
-Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC
-  [Host]  : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
-  LongRun : extra output line
+Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC
+.NET Core SDK 1.0.x.mock
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
 
Job=LongRun  IterationCount=100  LaunchCount=3  
 WarmupCount=15
- + - - + +
MethodMeanErrorStdDevP67
MethodMeanErrorStdDevP67CacheMisses
Foo1.000 nsNA0.000 ns1.000 ns
Bar1.000 nsNA0.000 ns1.000 ns
Foo1.000 ns0.000 ns0.000 ns1.000 ns7
Bar1.000 ns0.000 ns0.000 ns1.000 ns7
@@ -71,7 +71,7 @@ JsonExporter-brief "HasAttachedDebugger":false, "HasRyuJit":true, "Configuration":"CONFIGURATION", - "DotNetCliVersion":null, + "DotNetCliVersion":"1.0.x.mock", "ChronometerFrequency":{ "Hertz":2531248 }, @@ -89,9 +89,9 @@ JsonExporter-brief "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -116,13 +116,13 @@ JsonExporter-brief "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -154,9 +154,9 @@ JsonExporter-brief "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -181,13 +181,13 @@ JsonExporter-brief "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -214,7 +214,7 @@ JsonExporter-brief ############################################ JsonExporter-brief-compressed ############################################ -{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":null,"ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}}]} +{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":"1.0.x.mock","ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}}]} ############################################ JsonExporter-full ############################################ @@ -233,7 +233,7 @@ JsonExporter-full "HasAttachedDebugger":false, "HasRyuJit":true, "Configuration":"CONFIGURATION", - "DotNetCliVersion":null, + "DotNetCliVersion":"1.0.x.mock", "ChronometerFrequency":{ "Hertz":2531248 }, @@ -251,9 +251,9 @@ JsonExporter-full "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -278,13 +278,13 @@ JsonExporter-full "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -313,6 +313,56 @@ JsonExporter-full "IterationIndex":1, "Operations":1, "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":2, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":3, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":4, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":5, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":6, + "Operations":1, + "Nanoseconds":1 + } + ], + "Metrics":[ + { + "Value":7, + "Descriptor":{ + "Id":"CacheMisses", + "DisplayName":"CacheMisses", + "Legend":"Hardware counter 'CacheMisses' per single operation", + "NumberFormat":"N0", + "UnitType":0, + "Unit":null, + "TheGreaterTheBetter":false, + "PriorityInCategory":0 + } } ] },{ @@ -326,9 +376,9 @@ JsonExporter-full "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -353,13 +403,13 @@ JsonExporter-full "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -388,6 +438,56 @@ JsonExporter-full "IterationIndex":1, "Operations":1, "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":2, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":3, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":4, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":5, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":6, + "Operations":1, + "Nanoseconds":1 + } + ], + "Metrics":[ + { + "Value":7, + "Descriptor":{ + "Id":"CacheMisses", + "DisplayName":"CacheMisses", + "Legend":"Hardware counter 'CacheMisses' per single operation", + "NumberFormat":"N0", + "UnitType":0, + "Unit":null, + "TheGreaterTheBetter":false, + "PriorityInCategory":0 + } } ] } @@ -396,24 +496,24 @@ JsonExporter-full ############################################ JsonExporter-full-compressed ############################################ -{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":null,"ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1}]},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1}]}]} +{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":"1.0.x.mock","ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":2,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":3,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":4,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":5,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":6,"Operations":1,"Nanoseconds":1}],"Metrics":[{"Value":7,"Descriptor":{"Id":"CacheMisses","DisplayName":"CacheMisses","Legend":"Hardware counter 'CacheMisses' per single operation","NumberFormat":"N0","UnitType":0,"Unit":null,"TheGreaterTheBetter":false,"PriorityInCategory":0}}]},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":2,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":3,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":4,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":5,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":6,"Operations":1,"Nanoseconds":1}],"Metrics":[{"Value":7,"Descriptor":{"Id":"CacheMisses","DisplayName":"CacheMisses","Legend":"Hardware counter 'CacheMisses' per single operation","NumberFormat":"N0","UnitType":0,"Unit":null,"TheGreaterTheBetter":false,"PriorityInCategory":0}}]}]} ############################################ MarkdownExporter-default ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 - Method | Mean | Error | StdDev | P67 | -------- |---------:|------:|---------:|---------:| - Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | - Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | + Method | Mean | Error | StdDev | P67 | CacheMisses | +------- |---------:|---------:|---------:|---------:|------------:| + Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | + Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-atlassian ############################################ @@ -421,34 +521,34 @@ MarkdownExporter-atlassian BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 {noformat} -||Method ||Mean ||Error ||StdDev ||P67 || -| Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | -| Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | +||Method ||Mean ||Error ||StdDev ||P67 ||CacheMisses || +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-console ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 -| Method | Mean | Error | StdDev | P67 | -|------- |---------:|------:|---------:|---------:| -| Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | -| Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | +| Method | Mean | Error | StdDev | P67 | CacheMisses | +|------- |---------:|---------:|---------:|---------:|------------:| +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-github ############################################ @@ -456,63 +556,73 @@ MarkdownExporter-github BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 ``` -| Method | Mean | Error | StdDev | P67 | -|------- |---------:|------:|---------:|---------:| -| Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | -| Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | +| Method | Mean | Error | StdDev | P67 | CacheMisses | +|------- |---------:|---------:|---------:|---------:|------------:| +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-stackoverflow ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores - Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line + Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC + .NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 - Method | Mean | Error | StdDev | P67 | - ------- |---------:|------:|---------:|---------:| - Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | - Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | + Method | Mean | Error | StdDev | P67 | CacheMisses | + ------- |---------:|---------:|---------:|---------:|------------:| + Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | + Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ PlainExporter ############################################ *** MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15) *** * Raw * WorkloadResult 1: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 2: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 3: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 4: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 5: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 6: 1 op, 1.00 ns, 1.0000 ns/op * Statistics for WorkloadResult -Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 1, StdDev = 0.000 ns +Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 6, StdDev = 0.000 ns Min = 1.000 ns, Q1 = 1.000 ns, Median = 1.000 ns, Q3 = 1.000 ns, Max = 1.000 ns IQR = 0.000 ns, LowerFence = 1.000 ns, UpperFence = 1.000 ns -ConfidenceInterval = [NaN ns; NaN ns] (CI 99.9%), Margin = NaN ns (NaN% of Mean) +ConfidenceInterval = [1.000 ns; 1.000 ns] (CI 99.9%), Margin = 0.000 ns (0.00% of Mean) Skewness = NaN, Kurtosis = NaN, MValue = 2 -------------------- Histogram -------------------- -[0.500 ns ; 1.500 ns) | @ +[0.500 ns ; 1.500 ns) | @@@@@@ --------------------------------------------------- *** MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15) *** * Raw * WorkloadResult 1: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 2: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 3: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 4: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 5: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 6: 1 op, 1.00 ns, 1.0000 ns/op * Statistics for WorkloadResult -Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 1, StdDev = 0.000 ns +Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 6, StdDev = 0.000 ns Min = 1.000 ns, Q1 = 1.000 ns, Median = 1.000 ns, Q3 = 1.000 ns, Max = 1.000 ns IQR = 0.000 ns, LowerFence = 1.000 ns, UpperFence = 1.000 ns -ConfidenceInterval = [NaN ns; NaN ns] (CI 99.9%), Margin = NaN ns (NaN% of Mean) +ConfidenceInterval = [1.000 ns; 1.000 ns] (CI 99.9%), Margin = 0.000 ns (0.00% of Mean) Skewness = NaN, Kurtosis = NaN, MValue = 2 -------------------- Histogram -------------------- -[0.500 ns ; 1.500 ns) | @ +[0.500 ns ; 1.500 ns) | @@@@@@ --------------------------------------------------- ############################################ XmlExporter-brief @@ -533,6 +643,7 @@ XmlExporter-brief False True CONFIGURATION + 1.0.x.mock 2531248 @@ -548,8 +659,13 @@ XmlExporter-brief 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -565,13 +681,13 @@ XmlExporter-brief NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -585,6 +701,20 @@ XmlExporter-brief 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -602,8 +732,13 @@ XmlExporter-brief 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -619,13 +754,13 @@ XmlExporter-brief NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -639,6 +774,20 @@ XmlExporter-brief 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -652,7 +801,7 @@ XmlExporter-brief ############################################ XmlExporter-brief-compressed ############################################ -MockSummaryBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo11111111110000NaNNaN110L999NaNNaNNaN1111111110000MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar11111111110000NaNNaN110L999NaNNaNNaN1111111110000 +MockSummaryBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION1.0.x.mock2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000 ############################################ XmlExporter-full ############################################ @@ -672,6 +821,7 @@ XmlExporter-full False True CONFIGURATION + 1.0.x.mock 2531248 @@ -687,8 +837,13 @@ XmlExporter-full 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -704,13 +859,13 @@ XmlExporter-full NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -724,6 +879,20 @@ XmlExporter-full 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -740,6 +909,46 @@ XmlExporter-full 1 1 + + Workload + Result + 1 + 2 + 1 + 1 + + + Workload + Result + 1 + 3 + 1 + 1 + + + Workload + Result + 1 + 4 + 1 + 1 + + + Workload + Result + 1 + 5 + 1 + 1 + + + Workload + Result + 1 + 6 + 1 + 1 + @@ -751,8 +960,13 @@ XmlExporter-full 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -768,13 +982,13 @@ XmlExporter-full NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -788,6 +1002,20 @@ XmlExporter-full 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -804,6 +1032,46 @@ XmlExporter-full 1 1 + + Workload + Result + 1 + 2 + 1 + 1 + + + Workload + Result + 1 + 3 + 1 + 1 + + + Workload + Result + 1 + 4 + 1 + 1 + + + Workload + Result + 1 + 5 + 1 + 1 + + + Workload + Result + 1 + 6 + 1 + 1 + @@ -811,4 +1079,4 @@ XmlExporter-full ############################################ XmlExporter-full-compressed ############################################ -MockSummaryBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo11111111110000NaNNaN110L999NaNNaNNaN1111111110000WorkloadResult1111MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar11111111110000NaNNaN110L999NaNNaNNaN1111111110000WorkloadResult1111 +MockSummaryBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION1.0.x.mock2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000WorkloadResult1111WorkloadResult1211WorkloadResult1311WorkloadResult1411WorkloadResult1511WorkloadResult1611MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000WorkloadResult1111WorkloadResult1211WorkloadResult1311WorkloadResult1411WorkloadResult1511WorkloadResult1611 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_en-US.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_en-US.verified.txt index cd3482c04f..b406e20dd1 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_en-US.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_en-US.verified.txt @@ -4,18 +4,18 @@ AsciiDocExporter .... BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 .... [options="header"] |=== -|Method |Mean |Error |StdDev |P67 -|Foo | 1.000 ns| NA| 0.000 ns| 1.000 ns -|Bar | 1.000 ns| NA| 0.000 ns| 1.000 ns +|Method |Mean |Error |StdDev |P67 |CacheMisses +|Foo | 1.000 ns| 0.000 ns| 0.000 ns| 1.000 ns| 7 +|Bar | 1.000 ns| 0.000 ns| 0.000 ns| 1.000 ns| 7 |=== ############################################ HtmlExporter @@ -37,19 +37,19 @@ HtmlExporter 

 BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V)
 MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
-Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC
-  [Host]  : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
-  LongRun : extra output line
+Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC
+.NET Core SDK 1.0.x.mock
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
 
Job=LongRun  IterationCount=100  LaunchCount=3  
 WarmupCount=15
- + - - + +
MethodMeanErrorStdDevP67
MethodMeanErrorStdDevP67CacheMisses
Foo1.000 nsNA0.000 ns1.000 ns
Bar1.000 nsNA0.000 ns1.000 ns
Foo1.000 ns0.000 ns0.000 ns1.000 ns7
Bar1.000 ns0.000 ns0.000 ns1.000 ns7
@@ -71,7 +71,7 @@ JsonExporter-brief "HasAttachedDebugger":false, "HasRyuJit":true, "Configuration":"CONFIGURATION", - "DotNetCliVersion":null, + "DotNetCliVersion":"1.0.x.mock", "ChronometerFrequency":{ "Hertz":2531248 }, @@ -89,9 +89,9 @@ JsonExporter-brief "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -116,13 +116,13 @@ JsonExporter-brief "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -154,9 +154,9 @@ JsonExporter-brief "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -181,13 +181,13 @@ JsonExporter-brief "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -214,7 +214,7 @@ JsonExporter-brief ############################################ JsonExporter-brief-compressed ############################################ -{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":null,"ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}}]} +{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":"1.0.x.mock","ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}}]} ############################################ JsonExporter-full ############################################ @@ -233,7 +233,7 @@ JsonExporter-full "HasAttachedDebugger":false, "HasRyuJit":true, "Configuration":"CONFIGURATION", - "DotNetCliVersion":null, + "DotNetCliVersion":"1.0.x.mock", "ChronometerFrequency":{ "Hertz":2531248 }, @@ -251,9 +251,9 @@ JsonExporter-full "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -278,13 +278,13 @@ JsonExporter-full "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -313,6 +313,56 @@ JsonExporter-full "IterationIndex":1, "Operations":1, "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":2, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":3, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":4, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":5, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":6, + "Operations":1, + "Nanoseconds":1 + } + ], + "Metrics":[ + { + "Value":7, + "Descriptor":{ + "Id":"CacheMisses", + "DisplayName":"CacheMisses", + "Legend":"Hardware counter 'CacheMisses' per single operation", + "NumberFormat":"N0", + "UnitType":0, + "Unit":null, + "TheGreaterTheBetter":false, + "PriorityInCategory":0 + } } ] },{ @@ -326,9 +376,9 @@ JsonExporter-full "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -353,13 +403,13 @@ JsonExporter-full "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -388,6 +438,56 @@ JsonExporter-full "IterationIndex":1, "Operations":1, "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":2, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":3, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":4, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":5, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":6, + "Operations":1, + "Nanoseconds":1 + } + ], + "Metrics":[ + { + "Value":7, + "Descriptor":{ + "Id":"CacheMisses", + "DisplayName":"CacheMisses", + "Legend":"Hardware counter 'CacheMisses' per single operation", + "NumberFormat":"N0", + "UnitType":0, + "Unit":null, + "TheGreaterTheBetter":false, + "PriorityInCategory":0 + } } ] } @@ -396,24 +496,24 @@ JsonExporter-full ############################################ JsonExporter-full-compressed ############################################ -{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":null,"ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1}]},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1}]}]} +{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":"1.0.x.mock","ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":2,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":3,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":4,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":5,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":6,"Operations":1,"Nanoseconds":1}],"Metrics":[{"Value":7,"Descriptor":{"Id":"CacheMisses","DisplayName":"CacheMisses","Legend":"Hardware counter 'CacheMisses' per single operation","NumberFormat":"N0","UnitType":0,"Unit":null,"TheGreaterTheBetter":false,"PriorityInCategory":0}}]},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":2,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":3,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":4,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":5,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":6,"Operations":1,"Nanoseconds":1}],"Metrics":[{"Value":7,"Descriptor":{"Id":"CacheMisses","DisplayName":"CacheMisses","Legend":"Hardware counter 'CacheMisses' per single operation","NumberFormat":"N0","UnitType":0,"Unit":null,"TheGreaterTheBetter":false,"PriorityInCategory":0}}]}]} ############################################ MarkdownExporter-default ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 - Method | Mean | Error | StdDev | P67 | -------- |---------:|------:|---------:|---------:| - Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | - Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | + Method | Mean | Error | StdDev | P67 | CacheMisses | +------- |---------:|---------:|---------:|---------:|------------:| + Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | + Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-atlassian ############################################ @@ -421,34 +521,34 @@ MarkdownExporter-atlassian BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 {noformat} -||Method ||Mean ||Error ||StdDev ||P67 || -| Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | -| Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | +||Method ||Mean ||Error ||StdDev ||P67 ||CacheMisses || +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-console ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 -| Method | Mean | Error | StdDev | P67 | -|------- |---------:|------:|---------:|---------:| -| Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | -| Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | +| Method | Mean | Error | StdDev | P67 | CacheMisses | +|------- |---------:|---------:|---------:|---------:|------------:| +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-github ############################################ @@ -456,63 +556,73 @@ MarkdownExporter-github BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 ``` -| Method | Mean | Error | StdDev | P67 | -|------- |---------:|------:|---------:|---------:| -| Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | -| Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | +| Method | Mean | Error | StdDev | P67 | CacheMisses | +|------- |---------:|---------:|---------:|---------:|------------:| +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-stackoverflow ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores - Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line + Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC + .NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 - Method | Mean | Error | StdDev | P67 | - ------- |---------:|------:|---------:|---------:| - Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | - Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | + Method | Mean | Error | StdDev | P67 | CacheMisses | + ------- |---------:|---------:|---------:|---------:|------------:| + Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | + Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ PlainExporter ############################################ *** MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15) *** * Raw * WorkloadResult 1: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 2: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 3: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 4: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 5: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 6: 1 op, 1.00 ns, 1.0000 ns/op * Statistics for WorkloadResult -Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 1, StdDev = 0.000 ns +Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 6, StdDev = 0.000 ns Min = 1.000 ns, Q1 = 1.000 ns, Median = 1.000 ns, Q3 = 1.000 ns, Max = 1.000 ns IQR = 0.000 ns, LowerFence = 1.000 ns, UpperFence = 1.000 ns -ConfidenceInterval = [NaN ns; NaN ns] (CI 99.9%), Margin = NaN ns (NaN% of Mean) +ConfidenceInterval = [1.000 ns; 1.000 ns] (CI 99.9%), Margin = 0.000 ns (0.00% of Mean) Skewness = NaN, Kurtosis = NaN, MValue = 2 -------------------- Histogram -------------------- -[0.500 ns ; 1.500 ns) | @ +[0.500 ns ; 1.500 ns) | @@@@@@ --------------------------------------------------- *** MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15) *** * Raw * WorkloadResult 1: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 2: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 3: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 4: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 5: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 6: 1 op, 1.00 ns, 1.0000 ns/op * Statistics for WorkloadResult -Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 1, StdDev = 0.000 ns +Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 6, StdDev = 0.000 ns Min = 1.000 ns, Q1 = 1.000 ns, Median = 1.000 ns, Q3 = 1.000 ns, Max = 1.000 ns IQR = 0.000 ns, LowerFence = 1.000 ns, UpperFence = 1.000 ns -ConfidenceInterval = [NaN ns; NaN ns] (CI 99.9%), Margin = NaN ns (NaN% of Mean) +ConfidenceInterval = [1.000 ns; 1.000 ns] (CI 99.9%), Margin = 0.000 ns (0.00% of Mean) Skewness = NaN, Kurtosis = NaN, MValue = 2 -------------------- Histogram -------------------- -[0.500 ns ; 1.500 ns) | @ +[0.500 ns ; 1.500 ns) | @@@@@@ --------------------------------------------------- ############################################ XmlExporter-brief @@ -533,6 +643,7 @@ XmlExporter-brief False True CONFIGURATION + 1.0.x.mock 2531248 @@ -548,8 +659,13 @@ XmlExporter-brief 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -565,13 +681,13 @@ XmlExporter-brief NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -585,6 +701,20 @@ XmlExporter-brief 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -602,8 +732,13 @@ XmlExporter-brief 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -619,13 +754,13 @@ XmlExporter-brief NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -639,6 +774,20 @@ XmlExporter-brief 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -652,7 +801,7 @@ XmlExporter-brief ############################################ XmlExporter-brief-compressed ############################################ -MockSummaryBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo11111111110000NaNNaN110L999NaNNaNNaN1111111110000MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar11111111110000NaNNaN110L999NaNNaNNaN1111111110000 +MockSummaryBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION1.0.x.mock2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000 ############################################ XmlExporter-full ############################################ @@ -672,6 +821,7 @@ XmlExporter-full False True CONFIGURATION + 1.0.x.mock 2531248 @@ -687,8 +837,13 @@ XmlExporter-full 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -704,13 +859,13 @@ XmlExporter-full NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -724,6 +879,20 @@ XmlExporter-full 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -740,6 +909,46 @@ XmlExporter-full 1 1 + + Workload + Result + 1 + 2 + 1 + 1 + + + Workload + Result + 1 + 3 + 1 + 1 + + + Workload + Result + 1 + 4 + 1 + 1 + + + Workload + Result + 1 + 5 + 1 + 1 + + + Workload + Result + 1 + 6 + 1 + 1 + @@ -751,8 +960,13 @@ XmlExporter-full 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -768,13 +982,13 @@ XmlExporter-full NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -788,6 +1002,20 @@ XmlExporter-full 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -804,6 +1032,46 @@ XmlExporter-full 1 1 + + Workload + Result + 1 + 2 + 1 + 1 + + + Workload + Result + 1 + 3 + 1 + 1 + + + Workload + Result + 1 + 4 + 1 + 1 + + + Workload + Result + 1 + 5 + 1 + 1 + + + Workload + Result + 1 + 6 + 1 + 1 + @@ -811,4 +1079,4 @@ XmlExporter-full ############################################ XmlExporter-full-compressed ############################################ -MockSummaryBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo11111111110000NaNNaN110L999NaNNaNNaN1111111110000WorkloadResult1111MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar11111111110000NaNNaN110L999NaNNaNNaN1111111110000WorkloadResult1111 +MockSummaryBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION1.0.x.mock2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000WorkloadResult1111WorkloadResult1211WorkloadResult1311WorkloadResult1411WorkloadResult1511WorkloadResult1611MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000WorkloadResult1111WorkloadResult1211WorkloadResult1311WorkloadResult1411WorkloadResult1511WorkloadResult1611 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_ru-RU.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_ru-RU.verified.txt index bbfaff1da8..b406e20dd1 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_ru-RU.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_ru-RU.verified.txt @@ -4,18 +4,18 @@ AsciiDocExporter .... BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 .... [options="header"] |=== -|Method |Mean |Error |StdDev |P67 -|Foo | 1,000 ns| NA| 0,000 ns| 1,000 ns -|Bar | 1,000 ns| NA| 0,000 ns| 1,000 ns +|Method |Mean |Error |StdDev |P67 |CacheMisses +|Foo | 1.000 ns| 0.000 ns| 0.000 ns| 1.000 ns| 7 +|Bar | 1.000 ns| 0.000 ns| 0.000 ns| 1.000 ns| 7 |=== ############################################ HtmlExporter @@ -37,19 +37,19 @@ HtmlExporter 

 BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V)
 MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
-Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC
-  [Host]  : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
-  LongRun : extra output line
+Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC
+.NET Core SDK 1.0.x.mock
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
 
Job=LongRun  IterationCount=100  LaunchCount=3  
 WarmupCount=15
- + - - + +
MethodMeanErrorStdDevP67
MethodMeanErrorStdDevP67CacheMisses
Foo1,000 nsNA0,000 ns1,000 ns
Bar1,000 nsNA0,000 ns1,000 ns
Foo1.000 ns0.000 ns0.000 ns1.000 ns7
Bar1.000 ns0.000 ns0.000 ns1.000 ns7
@@ -71,7 +71,7 @@ JsonExporter-brief "HasAttachedDebugger":false, "HasRyuJit":true, "Configuration":"CONFIGURATION", - "DotNetCliVersion":null, + "DotNetCliVersion":"1.0.x.mock", "ChronometerFrequency":{ "Hertz":2531248 }, @@ -89,9 +89,9 @@ JsonExporter-brief "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -116,13 +116,13 @@ JsonExporter-brief "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -154,9 +154,9 @@ JsonExporter-brief "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -181,13 +181,13 @@ JsonExporter-brief "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -214,7 +214,7 @@ JsonExporter-brief ############################################ JsonExporter-brief-compressed ############################################ -{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":null,"ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}}]} +{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":"1.0.x.mock","ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}}]} ############################################ JsonExporter-full ############################################ @@ -233,7 +233,7 @@ JsonExporter-full "HasAttachedDebugger":false, "HasRyuJit":true, "Configuration":"CONFIGURATION", - "DotNetCliVersion":null, + "DotNetCliVersion":"1.0.x.mock", "ChronometerFrequency":{ "Hertz":2531248 }, @@ -251,9 +251,9 @@ JsonExporter-full "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -278,13 +278,13 @@ JsonExporter-full "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -313,6 +313,56 @@ JsonExporter-full "IterationIndex":1, "Operations":1, "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":2, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":3, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":4, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":5, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":6, + "Operations":1, + "Nanoseconds":1 + } + ], + "Metrics":[ + { + "Value":7, + "Descriptor":{ + "Id":"CacheMisses", + "DisplayName":"CacheMisses", + "Legend":"Hardware counter 'CacheMisses' per single operation", + "NumberFormat":"N0", + "UnitType":0, + "Unit":null, + "TheGreaterTheBetter":false, + "PriorityInCategory":0 + } } ] },{ @@ -326,9 +376,9 @@ JsonExporter-full "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -353,13 +403,13 @@ JsonExporter-full "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -388,6 +438,56 @@ JsonExporter-full "IterationIndex":1, "Operations":1, "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":2, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":3, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":4, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":5, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":6, + "Operations":1, + "Nanoseconds":1 + } + ], + "Metrics":[ + { + "Value":7, + "Descriptor":{ + "Id":"CacheMisses", + "DisplayName":"CacheMisses", + "Legend":"Hardware counter 'CacheMisses' per single operation", + "NumberFormat":"N0", + "UnitType":0, + "Unit":null, + "TheGreaterTheBetter":false, + "PriorityInCategory":0 + } } ] } @@ -396,24 +496,24 @@ JsonExporter-full ############################################ JsonExporter-full-compressed ############################################ -{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":null,"ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1}]},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1}]}]} +{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":"1.0.x.mock","ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":2,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":3,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":4,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":5,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":6,"Operations":1,"Nanoseconds":1}],"Metrics":[{"Value":7,"Descriptor":{"Id":"CacheMisses","DisplayName":"CacheMisses","Legend":"Hardware counter 'CacheMisses' per single operation","NumberFormat":"N0","UnitType":0,"Unit":null,"TheGreaterTheBetter":false,"PriorityInCategory":0}}]},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":2,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":3,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":4,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":5,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":6,"Operations":1,"Nanoseconds":1}],"Metrics":[{"Value":7,"Descriptor":{"Id":"CacheMisses","DisplayName":"CacheMisses","Legend":"Hardware counter 'CacheMisses' per single operation","NumberFormat":"N0","UnitType":0,"Unit":null,"TheGreaterTheBetter":false,"PriorityInCategory":0}}]}]} ############################################ MarkdownExporter-default ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 - Method | Mean | Error | StdDev | P67 | -------- |---------:|------:|---------:|---------:| - Foo | 1,000 ns | NA | 0,000 ns | 1,000 ns | - Bar | 1,000 ns | NA | 0,000 ns | 1,000 ns | + Method | Mean | Error | StdDev | P67 | CacheMisses | +------- |---------:|---------:|---------:|---------:|------------:| + Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | + Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-atlassian ############################################ @@ -421,34 +521,34 @@ MarkdownExporter-atlassian BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 {noformat} -||Method ||Mean ||Error ||StdDev ||P67 || -| Foo | 1,000 ns | NA | 0,000 ns | 1,000 ns | -| Bar | 1,000 ns | NA | 0,000 ns | 1,000 ns | +||Method ||Mean ||Error ||StdDev ||P67 ||CacheMisses || +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-console ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 -| Method | Mean | Error | StdDev | P67 | -|------- |---------:|------:|---------:|---------:| -| Foo | 1,000 ns | NA | 0,000 ns | 1,000 ns | -| Bar | 1,000 ns | NA | 0,000 ns | 1,000 ns | +| Method | Mean | Error | StdDev | P67 | CacheMisses | +|------- |---------:|---------:|---------:|---------:|------------:| +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-github ############################################ @@ -456,63 +556,73 @@ MarkdownExporter-github BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 ``` -| Method | Mean | Error | StdDev | P67 | -|------- |---------:|------:|---------:|---------:| -| Foo | 1,000 ns | NA | 0,000 ns | 1,000 ns | -| Bar | 1,000 ns | NA | 0,000 ns | 1,000 ns | +| Method | Mean | Error | StdDev | P67 | CacheMisses | +|------- |---------:|---------:|---------:|---------:|------------:| +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-stackoverflow ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores - Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line + Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC + .NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 - Method | Mean | Error | StdDev | P67 | - ------- |---------:|------:|---------:|---------:| - Foo | 1,000 ns | NA | 0,000 ns | 1,000 ns | - Bar | 1,000 ns | NA | 0,000 ns | 1,000 ns | + Method | Mean | Error | StdDev | P67 | CacheMisses | + ------- |---------:|---------:|---------:|---------:|------------:| + Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | + Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ PlainExporter ############################################ *** MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15) *** * Raw * WorkloadResult 1: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 2: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 3: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 4: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 5: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 6: 1 op, 1.00 ns, 1.0000 ns/op * Statistics for WorkloadResult -Mean = 1,000 ns; StdErr = 0,000 ns (0,00%); N = 1; StdDev = 0,000 ns -Min = 1,000 ns; Q1 = 1,000 ns; Median = 1,000 ns; Q3 = 1,000 ns; Max = 1,000 ns -IQR = 0,000 ns; LowerFence = 1,000 ns; UpperFence = 1,000 ns -ConfidenceInterval = [не число ns; не число ns] (CI 99.9%); Margin = не число ns (не число% of Mean) -Skewness = не число; Kurtosis = не число; MValue = 2 +Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 6, StdDev = 0.000 ns +Min = 1.000 ns, Q1 = 1.000 ns, Median = 1.000 ns, Q3 = 1.000 ns, Max = 1.000 ns +IQR = 0.000 ns, LowerFence = 1.000 ns, UpperFence = 1.000 ns +ConfidenceInterval = [1.000 ns; 1.000 ns] (CI 99.9%), Margin = 0.000 ns (0.00% of Mean) +Skewness = NaN, Kurtosis = NaN, MValue = 2 -------------------- Histogram -------------------- -[0,500 ns ; 1,500 ns) | @ +[0.500 ns ; 1.500 ns) | @@@@@@ --------------------------------------------------- *** MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15) *** * Raw * WorkloadResult 1: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 2: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 3: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 4: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 5: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 6: 1 op, 1.00 ns, 1.0000 ns/op * Statistics for WorkloadResult -Mean = 1,000 ns; StdErr = 0,000 ns (0,00%); N = 1; StdDev = 0,000 ns -Min = 1,000 ns; Q1 = 1,000 ns; Median = 1,000 ns; Q3 = 1,000 ns; Max = 1,000 ns -IQR = 0,000 ns; LowerFence = 1,000 ns; UpperFence = 1,000 ns -ConfidenceInterval = [не число ns; не число ns] (CI 99.9%); Margin = не число ns (не число% of Mean) -Skewness = не число; Kurtosis = не число; MValue = 2 +Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 6, StdDev = 0.000 ns +Min = 1.000 ns, Q1 = 1.000 ns, Median = 1.000 ns, Q3 = 1.000 ns, Max = 1.000 ns +IQR = 0.000 ns, LowerFence = 1.000 ns, UpperFence = 1.000 ns +ConfidenceInterval = [1.000 ns; 1.000 ns] (CI 99.9%), Margin = 0.000 ns (0.00% of Mean) +Skewness = NaN, Kurtosis = NaN, MValue = 2 -------------------- Histogram -------------------- -[0,500 ns ; 1,500 ns) | @ +[0.500 ns ; 1.500 ns) | @@@@@@ --------------------------------------------------- ############################################ XmlExporter-brief @@ -533,6 +643,7 @@ XmlExporter-brief False True CONFIGURATION + 1.0.x.mock 2531248 @@ -548,8 +659,13 @@ XmlExporter-brief 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -565,13 +681,13 @@ XmlExporter-brief NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -585,6 +701,20 @@ XmlExporter-brief 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -602,8 +732,13 @@ XmlExporter-brief 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -619,13 +754,13 @@ XmlExporter-brief NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -639,6 +774,20 @@ XmlExporter-brief 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -652,7 +801,7 @@ XmlExporter-brief ############################################ XmlExporter-brief-compressed ############################################ -MockSummaryBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo11111111110000NaNNaN110L999NaNNaNNaN1111111110000MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar11111111110000NaNNaN110L999NaNNaNNaN1111111110000 +MockSummaryBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION1.0.x.mock2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000 ############################################ XmlExporter-full ############################################ @@ -672,6 +821,7 @@ XmlExporter-full False True CONFIGURATION + 1.0.x.mock 2531248 @@ -687,8 +837,13 @@ XmlExporter-full 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -704,13 +859,13 @@ XmlExporter-full NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -724,6 +879,20 @@ XmlExporter-full 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -740,6 +909,46 @@ XmlExporter-full 1 1 + + Workload + Result + 1 + 2 + 1 + 1 + + + Workload + Result + 1 + 3 + 1 + 1 + + + Workload + Result + 1 + 4 + 1 + 1 + + + Workload + Result + 1 + 5 + 1 + 1 + + + Workload + Result + 1 + 6 + 1 + 1 + @@ -751,8 +960,13 @@ XmlExporter-full 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -768,13 +982,13 @@ XmlExporter-full NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -788,6 +1002,20 @@ XmlExporter-full 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -804,6 +1032,46 @@ XmlExporter-full 1 1 + + Workload + Result + 1 + 2 + 1 + 1 + + + Workload + Result + 1 + 3 + 1 + 1 + + + Workload + Result + 1 + 4 + 1 + 1 + + + Workload + Result + 1 + 5 + 1 + 1 + + + Workload + Result + 1 + 6 + 1 + 1 + @@ -811,4 +1079,4 @@ XmlExporter-full ############################################ XmlExporter-full-compressed ############################################ -MockSummaryBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo11111111110000NaNNaN110L999NaNNaNNaN1111111110000WorkloadResult1111MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar11111111110000NaNNaN110L999NaNNaNNaN1111111110000WorkloadResult1111 +MockSummaryBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION1.0.x.mock2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000WorkloadResult1111WorkloadResult1211WorkloadResult1311WorkloadResult1411WorkloadResult1511WorkloadResult1611MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000WorkloadResult1111WorkloadResult1211WorkloadResult1311WorkloadResult1411WorkloadResult1511WorkloadResult1611 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Escape_ParamsAndArguments.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Escape_ParamsAndArguments.verified.txt index 78bde45ccb..715b73166b 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Escape_ParamsAndArguments.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Escape_ParamsAndArguments.verified.txt @@ -2,18 +2,18 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | StringParam | charArg | Mean | Error | StdDev | ------- |------------ |-------- |---------:|--------:|--------:| - Foo | \t | \t | 102.0 ns | 6.09 ns | 1.58 ns | ^ - Foo | \t | \n | 202.0 ns | 6.09 ns | 1.58 ns | ^ - Bar | \t | ? | 302.0 ns | 6.09 ns | 1.58 ns | ^ - Foo | \n | \t | 402.0 ns | 6.09 ns | 1.58 ns | ^ - Foo | \n | \n | 502.0 ns | 6.09 ns | 1.58 ns | ^ - Bar | \n | ? | 602.0 ns | 6.09 ns | 1.58 ns | ^ + Foo | \t | \t | 114.5 ns | 5.88 ns | 8.80 ns | ^ + Foo | \t | \n | 214.5 ns | 5.88 ns | 8.80 ns | ^ + Bar | \t | ? | 314.5 ns | 5.88 ns | 8.80 ns | ^ + Foo | \n | \t | 414.5 ns | 5.88 ns | 8.80 ns | ^ + Foo | \n | \n | 514.5 ns | 5.88 ns | 8.80 ns | ^ + Bar | \n | ? | 614.5 ns | 5.88 ns | 8.80 ns | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_HideColumns_TableMarkDown.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_HideColumns_TableMarkDown.verified.txt new file mode 100644 index 0000000000..5b932e42da --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_HideColumns_TableMarkDown.verified.txt @@ -0,0 +1,15 @@ +=== HideColumns_TableMarkDown === + +BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + +StdDev=8.80 ns + + Method | Mean | Error | +------- |---------:|--------:| + Foo | 114.5 ns | 5.88 ns | + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt index b690775469..314e1f16bd 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,11 +10,11 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |---------:|--------:|--------:|------:|--------:|-----:|---------------------------- |--------- | - Foo | Job1 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Invalid_TwoJobBaselines.Foo | Yes | - Foo | Job2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 2 | Invalid_TwoJobBaselines.Foo | Yes | + Foo | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | Invalid_TwoJobBaselines.Foo | Yes | + Foo | Job2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 2 | Invalid_TwoJobBaselines.Foo | Yes | | | | | | | | | | | - Bar | Job1 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Invalid_TwoJobBaselines.Bar | Yes | - Bar | Job2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | Invalid_TwoJobBaselines.Bar | Yes | + Bar | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | Invalid_TwoJobBaselines.Bar | Yes | + Bar | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.94 | 0.09 | 2 | Invalid_TwoJobBaselines.Bar | Yes | Errors: 2 * Only 1 job in a group can have "Baseline = true" applied to it, group Invalid_TwoJobBaselines.Foo in class Invalid_TwoJobBaselines has 2 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt index aaba6592c2..11fbd70a0b 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt @@ -2,15 +2,15 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Foo | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | DefaultJob | Yes | - Bar | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | DefaultJob | Yes | + Foo | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DefaultJob | Yes | + Bar | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DefaultJob | Yes | Errors: 1 * Only 1 benchmark method in a group can have "Baseline = true" applied to it, group DefaultJob in class Invalid_TwoMethodBaselines has 2 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt index e8793a9409..5c8cbaad01 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,13 +10,13 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |---------:|--------:|--------:|------:|--------:|-----:|----------------------------- |--------- | - Base | Job1 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsJobs.Base | Yes | - Base | Job2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 2 | JobBaseline_MethodsJobs.Base | No | + Base | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | JobBaseline_MethodsJobs.Base | Yes | + Base | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 2 | JobBaseline_MethodsJobs.Base | No | | | | | | | | | | | - Foo | Job1 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsJobs.Foo | Yes | - Foo | Job2 | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | 2 | JobBaseline_MethodsJobs.Foo | No | + Foo | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | JobBaseline_MethodsJobs.Foo | Yes | + Foo | Job2 | 514.5 ns | 5.88 ns | 8.80 ns | 2.40 | 0.11 | 2 | JobBaseline_MethodsJobs.Foo | No | | | | | | | | | | | - Bar | Job1 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsJobs.Bar | Yes | - Bar | Job2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | JobBaseline_MethodsJobs.Bar | No | + Bar | Job1 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | JobBaseline_MethodsJobs.Bar | Yes | + Bar | Job2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.96 | 0.06 | 2 | JobBaseline_MethodsJobs.Bar | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt index 479192b41e..4f5f37b6c7 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,22 +10,22 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------- |--------- | - Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | Yes | ^ - Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | No | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | Yes | ^ + Base | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | No | | | | | | | | | | | | - Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | Yes | - Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | No | + Foo | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | Yes | + Foo | Job2 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 2.40 | 0.11 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | No | | | | | | | | | | | | - Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | Yes | - Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | No | + Bar | Job1 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | Yes | + Bar | Job2 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.96 | 0.06 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | No | | | | | | | | | | | | - Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | Yes | ^ - Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.43 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | No | + Base | Job1 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | Yes | ^ + Base | Job2 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.42 | 0.02 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | No | | | | | | | | | | | | - Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | Yes | - Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.37 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | No | + Foo | Job1 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | Yes | + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.37 | 0.02 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | No | | | | | | | | | | | | - Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | Yes | - Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | No | + Bar | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | Yes | + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.33 | 0.02 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt index 2355618257..aa22c9caab 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt @@ -2,15 +2,15 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Base | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | DefaultJob | Yes | - Foo | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | DefaultJob | No | - Bar | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | DefaultJob | No | + Base | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DefaultJob | Yes | + Foo | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DefaultJob | No | + Bar | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | DefaultJob | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt index 1096235ebb..297f782355 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,12 +10,12 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Base | Job1 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Job1 | Yes | - Foo | Job1 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | Job1 | No | - Bar | Job1 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | Job1 | No | + Base | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | Job1 | Yes | + Foo | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | Job1 | No | + Bar | Job1 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | Job1 | No | | | | | | | | | | | - Base | Job2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Job2 | Yes | - Foo | Job2 | 502.0 ns | 6.09 ns | 1.58 ns | 1.25 | 0.00 | 2 | Job2 | No | - Bar | Job2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.50 | 0.00 | 3 | Job2 | No | + Base | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | Job2 | Yes | + Foo | Job2 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | Job2 | No | + Bar | Job2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | Job2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt index 2178ec1d94..e30ce8166a 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt @@ -2,19 +2,19 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |------ |---------:|--------:|--------:|------:|--------:|-----:|---------------------- |--------- | - Base | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-DefaultJob | Yes | ^ - Foo | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | [Param=2]-DefaultJob | No | - Bar | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | [Param=2]-DefaultJob | No | + Base | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | [Param=2]-DefaultJob | Yes | ^ + Foo | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | [Param=2]-DefaultJob | No | + Bar | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | [Param=2]-DefaultJob | No | | | | | | | | | | | - Base | 10 | 402.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-DefaultJob | Yes | ^ - Foo | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.25 | 0.00 | 2 | [Param=10]-DefaultJob | No | - Bar | 10 | 602.0 ns | 6.09 ns | 1.58 ns | 1.50 | 0.00 | 3 | [Param=10]-DefaultJob | No | + Base | 10 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | [Param=10]-DefaultJob | Yes | ^ + Foo | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | [Param=10]-DefaultJob | No | + Bar | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | [Param=10]-DefaultJob | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt index 879a228733..f85ce9e878 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,20 +10,20 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------- |--------- | - Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-Job1 | Yes | ^ - Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | [Param=2]-Job1 | No | - Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | [Param=2]-Job1 | No | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | [Param=2]-Job1 | Yes | ^ + Foo | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | [Param=2]-Job1 | No | + Bar | Job1 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | [Param=2]-Job1 | No | | | | | | | | | | | | - Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-Job2 | Yes | - Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 1.25 | 0.00 | 2 | [Param=2]-Job2 | No | - Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.50 | 0.00 | 3 | [Param=2]-Job2 | No | + Base | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | [Param=2]-Job2 | Yes | + Foo | Job2 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | [Param=2]-Job2 | No | + Bar | Job2 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | [Param=2]-Job2 | No | | | | | | | | | | | | - Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-Job1 | Yes | ^ - Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.14 | 0.00 | 2 | [Param=10]-Job1 | No | - Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 1.28 | 0.00 | 3 | [Param=10]-Job1 | No | + Base | Job1 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | [Param=10]-Job1 | Yes | ^ + Foo | Job1 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.14 | 0.02 | 2 | [Param=10]-Job1 | No | + Bar | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 1.28 | 0.02 | 3 | [Param=10]-Job1 | No | | | | | | | | | | | | - Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-Job2 | Yes | - Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.10 | 0.00 | 2 | [Param=10]-Job2 | No | - Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.20 | 0.00 | 3 | [Param=10]-Job2 | No | + Base | Job2 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | [Param=10]-Job2 | Yes | + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.10 | 0.01 | 2 | [Param=10]-Job2 | No | + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.20 | 0.01 | 3 | [Param=10]-Job2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt index 2e9f774250..e71eeca6b6 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,9 +10,9 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Foo | Job1 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | * | Yes | - Bar | Job1 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | * | No | - Foo | Job2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | * | No | - Bar | Job2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 4 | * | No | + Foo | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | * | Yes | + Bar | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | * | No | + Foo | Job2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | * | No | + Bar | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 4 | * | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt index fac0a1a063..30d92f0b9e 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,14 +10,14 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |------ |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Foo | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2] | Yes | ^ - Bar | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | [Param=2] | No | - Foo | Job2 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | [Param=2] | No | - Bar | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 4 | [Param=2] | No | + Foo | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | [Param=2] | Yes | ^ + Bar | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | [Param=2] | No | + Foo | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | [Param=2] | No | + Bar | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 4 | [Param=2] | No | | | | | | | | | | | | - Foo | Job1 | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10] | Yes | ^ - Bar | Job1 | 10 | 602.0 ns | 6.09 ns | 1.58 ns | 1.20 | 0.00 | 2 | [Param=10] | No | - Foo | Job2 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.40 | 0.00 | 3 | [Param=10] | No | - Bar | Job2 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.60 | 0.00 | 4 | [Param=10] | No | + Foo | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | [Param=10] | Yes | ^ + Bar | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.19 | 0.03 | 2 | [Param=10] | No | + Foo | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.39 | 0.03 | 3 | [Param=10] | No | + Bar | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.58 | 0.03 | 4 | [Param=10] | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs.verified.txt index 52a50e2c03..892e8d3538 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,17 +10,17 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|-----:|------------- |--------- | - Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1 | * | No | ^ - Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 2 | * | No | - Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 3 | * | No | - Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 4 | * | No | - Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 5 | * | No | - Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 6 | * | No | - Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 7 | * | No | ^ - Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 8 | * | No | - Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 9 | * | No | - Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 10 | * | No | - Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 11 | * | No | - Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 12 | * | No | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1 | * | No | ^ + Foo | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 2 | * | No | + Bar | Job1 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 3 | * | No | + Base | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 4 | * | No | + Foo | Job2 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 5 | * | No | + Bar | Job2 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 6 | * | No | + Base | Job1 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 7 | * | No | ^ + Foo | Job1 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 8 | * | No | + Bar | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 9 | * | No | + Base | Job2 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 10 | * | No | + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 11 | * | No | + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 12 | * | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt index 355aa0ebd1..a7a7350881 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,36 +10,36 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------------------------- |--------- | - A1 | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=2]-CatA | Yes | ^ + A1 | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=2]-CatA | Yes | ^ | | | | | | | | | | | - A1 | Job1 | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=10]-CatA | Yes | ^ + A1 | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=10]-CatA | Yes | ^ | | | | | | | | | | | - A1 | Job2 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=2]-CatA | Yes | ^ + A1 | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=2]-CatA | Yes | ^ | | | | | | | | | | | - A1 | Job2 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=10]-CatA | Yes | ^ + A1 | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=10]-CatA | Yes | ^ | | | | | | | | | | | - A2 | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=2]-CatA | No | ^ + A2 | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=2]-CatA | No | ^ | | | | | | | | | | | - A2 | Job1 | 10 | 602.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=10]-CatA | No | ^ + A2 | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=10]-CatA | No | ^ | | | | | | | | | | | - A2 | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=2]-CatA | No | ^ + A2 | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=2]-CatA | No | ^ | | | | | | | | | | | - A2 | Job2 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=10]-CatA | No | ^ + A2 | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=10]-CatA | No | ^ | | | | | | | | | | | - B1 | Job1 | 2 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=2]-CatB | Yes | ^ + B1 | Job1 | 2 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=2]-CatB | Yes | ^ | | | | | | | | | | | - B1 | Job1 | 10 | 1,302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=10]-CatB | Yes | ^ + B1 | Job1 | 10 | 1,314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=10]-CatB | Yes | ^ | | | | | | | | | | | - B1 | Job2 | 2 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=2]-CatB | Yes | ^ + B1 | Job2 | 2 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=2]-CatB | Yes | ^ | | | | | | | | | | | - B1 | Job2 | 10 | 1,502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=10]-CatB | Yes | ^ + B1 | Job2 | 10 | 1,514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=10]-CatB | Yes | ^ | | | | | | | | | | | - B2 | Job1 | 2 | 1,002.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=2]-CatB | No | ^ + B2 | Job1 | 2 | 1,014.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=2]-CatB | No | ^ | | | | | | | | | | | - B2 | Job1 | 10 | 1,402.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=10]-CatB | No | ^ + B2 | Job1 | 10 | 1,414.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=10]-CatB | No | ^ | | | | | | | | | | | - B2 | Job2 | 2 | 1,202.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=2]-CatB | No | ^ + B2 | Job2 | 2 | 1,214.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=2]-CatB | No | ^ | | | | | | | | | | | - B2 | Job2 | 10 | 1,602.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=10]-CatB | No | ^ + B2 | Job2 | 10 | 1,614.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=10]-CatB | No | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt index b586e21cd9..bb98037dd3 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,28 +10,28 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|--------------------- |--------- | - A1 | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-[Param=2]-Job1 | Yes | ^ - A2 | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | CatA-[Param=2]-Job1 | No | + A1 | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | CatA-[Param=2]-Job1 | Yes | ^ + A2 | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | CatA-[Param=2]-Job1 | No | | | | | | | | | | | | - A1 | Job2 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-[Param=2]-Job2 | Yes | - A2 | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | 2 | CatA-[Param=2]-Job2 | No | + A1 | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | CatA-[Param=2]-Job2 | Yes | + A2 | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.32 | 0.05 | 2 | CatA-[Param=2]-Job2 | No | | | | | | | | | | | | - A1 | Job1 | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-[Param=10]-Job1 | Yes | ^ - A2 | Job1 | 10 | 602.0 ns | 6.09 ns | 1.58 ns | 1.20 | 0.00 | 2 | CatA-[Param=10]-Job1 | No | + A1 | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | CatA-[Param=10]-Job1 | Yes | ^ + A2 | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.19 | 0.03 | 2 | CatA-[Param=10]-Job1 | No | | | | | | | | | | | | - A1 | Job2 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-[Param=10]-Job2 | Yes | - A2 | Job2 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.14 | 0.00 | 2 | CatA-[Param=10]-Job2 | No | + A1 | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | CatA-[Param=10]-Job2 | Yes | + A2 | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.14 | 0.02 | 2 | CatA-[Param=10]-Job2 | No | | | | | | | | | | | | - B1 | Job1 | 2 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-[Param=2]-Job1 | Yes | ^ - B2 | Job1 | 2 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.11 | 0.00 | 2 | CatB-[Param=2]-Job1 | No | + B1 | Job1 | 2 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-[Param=2]-Job1 | Yes | ^ + B2 | Job1 | 2 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.11 | 0.01 | 2 | CatB-[Param=2]-Job1 | No | | | | | | | | | | | | - B1 | Job2 | 2 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-[Param=2]-Job2 | Yes | - B2 | Job2 | 2 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.09 | 0.00 | 2 | CatB-[Param=2]-Job2 | No | + B1 | Job2 | 2 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-[Param=2]-Job2 | Yes | + B2 | Job2 | 2 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.09 | 0.01 | 2 | CatB-[Param=2]-Job2 | No | | | | | | | | | | | | - B1 | Job1 | 10 | 1,302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-[Param=10]-Job1 | Yes | ^ - B2 | Job1 | 10 | 1,402.0 ns | 6.09 ns | 1.58 ns | 1.08 | 0.00 | 2 | CatB-[Param=10]-Job1 | No | + B1 | Job1 | 10 | 1,314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-[Param=10]-Job1 | Yes | ^ + B2 | Job1 | 10 | 1,414.5 ns | 5.88 ns | 8.80 ns | 1.08 | 0.01 | 2 | CatB-[Param=10]-Job1 | No | | | | | | | | | | | | - B1 | Job2 | 10 | 1,502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-[Param=10]-Job2 | Yes | - B2 | Job2 | 10 | 1,602.0 ns | 6.09 ns | 1.58 ns | 1.07 | 0.00 | 2 | CatB-[Param=10]-Job2 | No | + B1 | Job2 | 10 | 1,514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-[Param=10]-Job2 | Yes | + B2 | Job2 | 10 | 1,614.5 ns | 5.88 ns | 8.80 ns | 1.07 | 0.01 | 2 | CatB-[Param=10]-Job2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByJob.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByJob.verified.txt index 9944b6dac1..d1330fd6c7 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByJob.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByJob.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,18 +10,18 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|-----:|------------- |--------- | - Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1 | Job1 | No | ^ - Base | Job1 | 10 | 302.0 ns | 6.09 ns | 1.58 ns | 2 | Job1 | No | ^ - Foo | Job1 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 3 | Job1 | No | ^ - Bar | Job1 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 4 | Job1 | No | - Foo | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 5 | Job1 | No | ^ - Bar | Job1 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 6 | Job1 | No | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1 | Job1 | No | ^ + Base | Job1 | 10 | 314.5 ns | 5.88 ns | 8.80 ns | 2 | Job1 | No | ^ + Foo | Job1 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 3 | Job1 | No | ^ + Bar | Job1 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 4 | Job1 | No | + Foo | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 5 | Job1 | No | ^ + Bar | Job1 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 6 | Job1 | No | | | | | | | | | | - Base | Job2 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1 | Job2 | No | ^ - Base | Job2 | 10 | 402.0 ns | 6.09 ns | 1.58 ns | 2 | Job2 | No | ^ - Foo | Job2 | 2 | 702.0 ns | 6.09 ns | 1.58 ns | 3 | Job2 | No | ^ - Bar | Job2 | 2 | 802.0 ns | 6.09 ns | 1.58 ns | 4 | Job2 | No | - Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 5 | Job2 | No | ^ - Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 6 | Job2 | No | + Base | Job2 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1 | Job2 | No | ^ + Base | Job2 | 10 | 414.5 ns | 5.88 ns | 8.80 ns | 2 | Job2 | No | ^ + Foo | Job2 | 2 | 714.5 ns | 5.88 ns | 8.80 ns | 3 | Job2 | No | ^ + Bar | Job2 | 2 | 814.5 ns | 5.88 ns | 8.80 ns | 4 | Job2 | No | + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 5 | Job2 | No | ^ + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 6 | Job2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByMethod.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByMethod.verified.txt index fd34bb1c63..8b5b634e07 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByMethod.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByMethod.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,19 +10,19 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|-----:|------------------------------------------------ |--------- | - Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | ^ - Base | Job2 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 2 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | - Base | Job1 | 10 | 302.0 ns | 6.09 ns | 1.58 ns | 3 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | ^ - Base | Job2 | 10 | 402.0 ns | 6.09 ns | 1.58 ns | 4 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | ^ + Base | Job2 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 2 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | + Base | Job1 | 10 | 314.5 ns | 5.88 ns | 8.80 ns | 3 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | ^ + Base | Job2 | 10 | 414.5 ns | 5.88 ns | 8.80 ns | 4 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | | | | | | | | | | - Foo | Job1 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 1 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | ^ - Foo | Job2 | 2 | 702.0 ns | 6.09 ns | 1.58 ns | 2 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | - Foo | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 3 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | ^ - Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 4 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | + Foo | Job1 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 1 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | ^ + Foo | Job2 | 2 | 714.5 ns | 5.88 ns | 8.80 ns | 2 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | + Foo | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 3 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | ^ + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 4 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | | | | | | | | | | - Bar | Job1 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | ^ - Bar | Job2 | 2 | 802.0 ns | 6.09 ns | 1.58 ns | 2 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | - Bar | Job1 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 3 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | ^ - Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 4 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | + Bar | Job1 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 1 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | ^ + Bar | Job2 | 2 | 814.5 ns | 5.88 ns | 8.80 ns | 2 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | + Bar | Job1 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 3 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | ^ + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 4 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt index 83dac489bd..6fa6a6534a 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,18 +10,18 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|-----:|------------- |--------- | - Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1 | [Param=2] | No | ^ - Base | Job2 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 2 | [Param=2] | No | - Foo | Job1 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 3 | [Param=2] | No | - Bar | Job1 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 4 | [Param=2] | No | - Foo | Job2 | 2 | 702.0 ns | 6.09 ns | 1.58 ns | 5 | [Param=2] | No | - Bar | Job2 | 2 | 802.0 ns | 6.09 ns | 1.58 ns | 6 | [Param=2] | No | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1 | [Param=2] | No | ^ + Base | Job2 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 2 | [Param=2] | No | + Foo | Job1 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 3 | [Param=2] | No | + Bar | Job1 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 4 | [Param=2] | No | + Foo | Job2 | 2 | 714.5 ns | 5.88 ns | 8.80 ns | 5 | [Param=2] | No | + Bar | Job2 | 2 | 814.5 ns | 5.88 ns | 8.80 ns | 6 | [Param=2] | No | | | | | | | | | | - Base | Job1 | 10 | 302.0 ns | 6.09 ns | 1.58 ns | 1 | [Param=10] | No | ^ - Base | Job2 | 10 | 402.0 ns | 6.09 ns | 1.58 ns | 2 | [Param=10] | No | - Foo | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 3 | [Param=10] | No | - Bar | Job1 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 4 | [Param=10] | No | - Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 5 | [Param=10] | No | - Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 6 | [Param=10] | No | + Base | Job1 | 10 | 314.5 ns | 5.88 ns | 8.80 ns | 1 | [Param=10] | No | ^ + Base | Job2 | 10 | 414.5 ns | 5.88 ns | 8.80 ns | 2 | [Param=10] | No | + Foo | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 3 | [Param=10] | No | + Bar | Job1 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 4 | [Param=10] | No | + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 5 | [Param=10] | No | + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 6 | [Param=10] | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/XmlSerializerTests.cs b/tests/BenchmarkDotNet.Tests/Exporters/XmlSerializerTests.cs index 902a6339bd..b254b7622b 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/XmlSerializerTests.cs +++ b/tests/BenchmarkDotNet.Tests/Exporters/XmlSerializerTests.cs @@ -101,7 +101,7 @@ public void CtorThrowsWhenParameterIsNull() [InlineData(null, typeof(ArgumentException))] [InlineData(" ", typeof(ArgumentException))] [InlineData("", typeof(ArgumentException))] - public void WithRootNameThrowsGivenNameIsNullOrWhiteSpace(string name, Type exception) + public void WithRootNameThrowsGivenNameIsNullOrWhiteSpace(string? name, Type exception) { Assert.Throws(exception, () => XmlSerializer.GetBuilder(typeof(MockSource)) .WithRootName(name) @@ -116,7 +116,7 @@ public void WithRootNameThrowsGivenNameIsNullOrWhiteSpace(string name, Type exce [InlineData(null, "MockItem", typeof(ArgumentException))] [InlineData(" ", "MockItem", typeof(ArgumentException))] [InlineData("", "MockItem", typeof(ArgumentException))] - public void WithCollectionItemNameThrowsGivenInvalidArguments(string collectionName, string itemName, Type exception) + public void WithCollectionItemNameThrowsGivenInvalidArguments(string? collectionName, string? itemName, Type exception) { Assert.Throws(exception, () => XmlSerializer.GetBuilder(typeof(MockSource)) .WithCollectionItemName(collectionName, itemName) @@ -127,7 +127,7 @@ public void WithCollectionItemNameThrowsGivenInvalidArguments(string collectionN [InlineData(null, typeof(ArgumentException))] [InlineData(" ", typeof(ArgumentException))] [InlineData("", typeof(ArgumentException))] - public void WithExcludedPropertyThrowsGivenNameIsNullOrWhiteSpace(string name, Type exception) + public void WithExcludedPropertyThrowsGivenNameIsNullOrWhiteSpace(string? name, Type exception) { Assert.Throws(exception, () => XmlSerializer.GetBuilder(typeof(MockSource)) .WithExcludedProperty(name) diff --git a/tests/BenchmarkDotNet.Tests/Infra/VerifyHelper.cs b/tests/BenchmarkDotNet.Tests/Infra/VerifyHelper.cs new file mode 100644 index 0000000000..9d563d8ca6 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Infra/VerifyHelper.cs @@ -0,0 +1,16 @@ +using VerifyTests; + +namespace BenchmarkDotNet.Tests.Infra; + +public static class VerifyHelper +{ + public static VerifySettings Create(string? typeName = null) + { + var result = new VerifySettings(); + result.UseDirectory("VerifiedFiles"); + result.DisableDiff(); + if (typeName != null) + result.UseTypeName(typeName); + return result; + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Jobs/JobIdGeneratorTests.cs b/tests/BenchmarkDotNet.Tests/Jobs/JobIdGeneratorTests.cs new file mode 100644 index 0000000000..c0e845bb7a --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Jobs/JobIdGeneratorTests.cs @@ -0,0 +1,27 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.CsProj; +using Xunit; + +namespace BenchmarkDotNet.Tests.Jobs; + +public class JobIdGeneratorTests +{ + [Theory] + [MemberData(nameof(GetTheoryData), DisableDiscoveryEnumeration = true)] + public void AutoGenerateJobId(string expectedId, Job job) + { + // Act + var result = job.ResolvedId; + + // Assert + Assert.Equal(expectedId, result); + } + + public static TheoryData GetTheoryData() => new TheoryData() + { + {"Job-OOTPKI", Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp80) }, + {"Job-QAODSR", Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp90) }, + {"Job-KHMDUZ", Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp80).WithRuntime(CoreRuntime.Core80) }, + }; +} diff --git a/tests/BenchmarkDotNet.Tests/KnownIssue.cs b/tests/BenchmarkDotNet.Tests/KnownIssue.cs index fe6cb83ee5..1831ad3b08 100644 --- a/tests/BenchmarkDotNet.Tests/KnownIssue.cs +++ b/tests/BenchmarkDotNet.Tests/KnownIssue.cs @@ -2,7 +2,7 @@ namespace BenchmarkDotNet.Tests { public class KnownIssue { - public static KnownIssue Issue2299 => new (2299, "Non-supported Mono on Linux", false); + public static KnownIssue Issue2299 => new(2299, "Non-supported Mono on Linux", false); public int Number { get; } public string Description { get; } diff --git a/tests/BenchmarkDotNet.Tests/LightJsonSerializerTests.cs b/tests/BenchmarkDotNet.Tests/LightJsonSerializerTests.cs new file mode 100644 index 0000000000..16ef22efed --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/LightJsonSerializerTests.cs @@ -0,0 +1,56 @@ +using System.Collections.Generic; +using System.Linq; +using SimpleJson; +using Xunit; + +namespace BenchmarkDotNet.Tests +{ + public class LightJsonSerializerTests + { + [Theory] + [InlineData(10.0, "test", double.NaN)] + public void SimpleJson_ReplaceUnsupportedNumericValues_Smoke(object val1, object val2, object val3) + { + //Arrange + var data = new List() + { + val1, val2, val3 + }; + + //Act + for (int i = 0; i < data.Count; i++) + { + data[i] = SimpleJsonSerializer.ReplaceUnsupportedNumericValues(data[i]); + } + + //Assert + Assert.Equal(val1, data[0]); + Assert.Equal(val2, data[1]); + Assert.Equal(SimpleJsonSerializer.JSON_EMPTY_STRING, data[2]); + } + + + [Fact] + public void SimpleJson_SerializeObjectWithUnsupportedNumericValues_ReturnsValidJson() + { + //Arrange + var data = new Dictionary + { + { "Statistic1", float.NaN}, + { "Statistic2", float.NegativeInfinity }, + { "Statistic3", float.PositiveInfinity }, + { "Statistic4", double.NaN}, + { "Statistic5", double.NegativeInfinity }, + { "Statistic6", double.PositiveInfinity } + }; + + //Act + string json = SimpleJsonSerializer.SerializeObject(data); + + //Assert + Assert.True(SimpleJsonSerializer.TryDeserializeObject(json, out var obj)); + var values = (obj as SimpleJson.JsonObject).Select(x => x.Value); + Assert.True(values.All(x => x.Equals(string.Empty))); + } + } +} diff --git a/tests/BenchmarkDotNet.Tests/Mathematics/RankTests.cs b/tests/BenchmarkDotNet.Tests/Mathematics/RankTests.cs index cb3fe7712a..2118b43be1 100644 --- a/tests/BenchmarkDotNet.Tests/Mathematics/RankTests.cs +++ b/tests/BenchmarkDotNet.Tests/Mathematics/RankTests.cs @@ -8,13 +8,13 @@ public class RankTests [Fact] public void RankTest() { - var s1 = new Statistics(100, 101, 100, 101); - var s2 = new Statistics(300, 301, 300, 301); - var s3 = new Statistics(200.3279, 200.3178, 200.4046); - var s4 = new Statistics(200.2298, 200.5738, 200.3582); - var s5 = new Statistics(195, 196, 195, 196); - var actualRanks = RankHelper.GetRanks(s1, s2, s3, s4, s5); - var expectedRanks = new[] { 1, 4, 3, 3, 2 }; + var s1 = new Statistics(100, 101, 100, 101, 100, 101, 100, 101, 100, 101, 100, 101, 100, 101, 100, 101, 100, 101, 100, 101); + var s2 = new Statistics(300, 301, 300, 301, 300, 301, 300, 301, 300, 301, 300, 301, 300, 301, 300, 301, 300, 301, 300, 301); + var s3 = new Statistics(200.3279, 200.3178, 200.4046, 200.3279, 200.3178, 200.4046, 200.3279, 200.3178, 200.4046, 200.3279, 200.3178, 200.4046); + var s4 = new Statistics(200.2298, 200.5738, 200.3582, 200.2298, 200.5738, 200.3582, 200.2298, 200.5738, 200.3582, 200.2298, 200.5738, 200.3582); + var s5 = new Statistics(195, 196, 195, 196, 195, 196, 195, 196, 195, 196, 195, 196, 195, 196, 195, 196, 195, 196, 195, 196); + int[] actualRanks = RankHelper.GetRanks(s1, s2, s3, s4, s5); + int[] expectedRanks = [1, 4, 3, 3, 2]; Assert.Equal(expectedRanks, actualRanks); } } diff --git a/tests/BenchmarkDotNet.Tests/Mathematics/StatisticsTests.cs b/tests/BenchmarkDotNet.Tests/Mathematics/StatisticsTests.cs index f9d21bf452..73843f0bc2 100644 --- a/tests/BenchmarkDotNet.Tests/Mathematics/StatisticsTests.cs +++ b/tests/BenchmarkDotNet.Tests/Mathematics/StatisticsTests.cs @@ -1,196 +1,201 @@ using System; +using System.Collections.Generic; using System.Linq; using BenchmarkDotNet.Mathematics; -using Perfolizer.Mathematics.Common; +using BenchmarkDotNet.Tests.Common; +using JetBrains.Annotations; using Xunit; using Xunit.Abstractions; -namespace BenchmarkDotNet.Tests.Mathematics +namespace BenchmarkDotNet.Tests.Mathematics; + +public class StatisticsTests(ITestOutputHelper output) { - public class StatisticsTests + private void Print(Statistics summary) { - private readonly ITestOutputHelper output; + output.WriteLine("N = " + summary.N); + output.WriteLine("Min = " + summary.Min); + output.WriteLine("LowerFence = " + summary.LowerFence); + output.WriteLine("Q1 = " + summary.Q1); + output.WriteLine("Median = " + summary.Median); + output.WriteLine("Mean = " + summary.Mean); + output.WriteLine("Q3 = " + summary.Q3); + output.WriteLine("UpperFence = " + summary.UpperFence); + output.WriteLine("Max = " + summary.Max); + output.WriteLine("InterquartileRange = " + summary.InterquartileRange); + output.WriteLine("StandardDeviation = " + summary.StandardDeviation); + output.WriteLine("Outlier = [" + string.Join("; ", summary.AllOutliers) + "]"); + output.WriteLine("CI = " + summary.PerfolizerConfidenceInterval.ToString(TestCultureInfo.Instance)); + output.WriteLine("Percentiles = " + summary.Percentiles.ToString(TestCultureInfo.Instance)); + } - public StatisticsTests(ITestOutputHelper output) - { - this.output = output; - } + [Fact] + public void StatisticsWithN0Test() + { + Assert.Throws(() => new Statistics()); + } - private void Print(Statistics summary) - { - output.WriteLine("N = " + summary.N); - output.WriteLine("Min = " + summary.Min); - output.WriteLine("LowerFence = " + summary.LowerFence); - output.WriteLine("Q1 = " + summary.Q1); - output.WriteLine("Median = " + summary.Median); - output.WriteLine("Mean = " + summary.Mean); - output.WriteLine("Q3 = " + summary.Q3); - output.WriteLine("UpperFence = " + summary.UpperFence); - output.WriteLine("Max = " + summary.Max); - output.WriteLine("InterquartileRange = " + summary.InterquartileRange); - output.WriteLine("StandardDeviation = " + summary.StandardDeviation); - output.WriteLine("Outlier = [" + string.Join("; ", summary.AllOutliers) + "]"); - output.WriteLine("CI = " + summary.ConfidenceInterval.ToString(TestCultureInfo.Instance)); - output.WriteLine("Percentiles = " + summary.Percentiles.ToString(TestCultureInfo.Instance)); - } + [Fact] + public void StatisticsWithN1Test() + { + var summary = new Statistics(1); + Print(summary); + AssertEqual(1, summary.Min); + AssertEqual(1, summary.LowerFence); + AssertEqual(1, summary.Q1); + AssertEqual(1, summary.Median); + AssertEqual(1, summary.Mean); + AssertEqual(1, summary.Q3); + AssertEqual(1, summary.UpperFence); + AssertEqual(1, summary.Max); + AssertEqual(0, summary.InterquartileRange); + AssertEqual(0, summary.StandardDeviation); + AssertEqual(Array.Empty(), summary.AllOutliers); + AssertEqual(1, summary.Percentiles.P0); + AssertEqual(1, summary.Percentiles.P25); + AssertEqual(1, summary.Percentiles.P50); + AssertEqual(1, summary.Percentiles.P85); + AssertEqual(1, summary.Percentiles.P95); + AssertEqual(1, summary.Percentiles.P100); + } - [Fact] - public void StatisticsWithN0Test() - { - Assert.Throws(() => new Statistics()); - } + [Fact] + public void StatisticsWithN2Test() + { + var summary = new Statistics(1, 2); + Print(summary); + AssertEqual(1, summary.Min); + AssertEqual(0.5, summary.LowerFence); + AssertEqual(1.25, summary.Q1); + AssertEqual(1.5, summary.Median); + AssertEqual(1.5, summary.Mean); + AssertEqual(1.75, summary.Q3); + AssertEqual(2.5, summary.UpperFence); + AssertEqual(2, summary.Max); + AssertEqual(0.5, summary.InterquartileRange); + AssertEqual(0.70711, summary.StandardDeviation, AbsoluteEqualityComparer.E4); + AssertEqual(Array.Empty(), summary.AllOutliers); + AssertEqual(1, summary.Percentiles.P0); + AssertEqual(1.25, summary.Percentiles.P25); + AssertEqual(1.5, summary.Percentiles.P50); + AssertEqual(1.85, summary.Percentiles.P85); + AssertEqual(1.95, summary.Percentiles.P95); + AssertEqual(2, summary.Percentiles.P100); + } - [Fact] - public void StatisticsWithN1Test() - { - var summary = new Statistics(1); - Print(summary); - Assert.Equal(1, summary.Min); - Assert.Equal(1, summary.LowerFence); - Assert.Equal(1, summary.Q1); - Assert.Equal(1, summary.Median); - Assert.Equal(1, summary.Mean); - Assert.Equal(1, summary.Q3); - Assert.Equal(1, summary.UpperFence); - Assert.Equal(1, summary.Max); - Assert.Equal(0, summary.InterquartileRange); - Assert.Equal(0, summary.StandardDeviation); - Assert.Equal(Array.Empty(), summary.AllOutliers); - Assert.Equal(1, summary.Percentiles.P0); - Assert.Equal(1, summary.Percentiles.P25); - Assert.Equal(1, summary.Percentiles.P50); - Assert.Equal(1, summary.Percentiles.P85); - Assert.Equal(1, summary.Percentiles.P95); - Assert.Equal(1, summary.Percentiles.P100); - } + [Fact] + public void StatisticsWithN3Test() + { + var summary = new Statistics(1, 2, 4); + Print(summary); + AssertEqual(1, summary.Min); + AssertEqual(-0.75, summary.LowerFence); + AssertEqual(1.5, summary.Q1); + AssertEqual(2, summary.Median); + AssertEqual(2.333333, summary.Mean, AbsoluteEqualityComparer.E4); + AssertEqual(3, summary.Q3); + AssertEqual(5.25, summary.UpperFence); + AssertEqual(4, summary.Max); + AssertEqual(1.5, summary.InterquartileRange); + AssertEqual(1.52753, summary.StandardDeviation, AbsoluteEqualityComparer.E4); + AssertEqual(Array.Empty(), summary.AllOutliers); + AssertEqual(1, summary.Percentiles.P0); + AssertEqual(1.5, summary.Percentiles.P25); + AssertEqual(2, summary.Percentiles.P50); + AssertEqual(3.4, summary.Percentiles.P85); + AssertEqual(3.8, summary.Percentiles.P95); + AssertEqual(4, summary.Percentiles.P100); + } - [Fact] - public void StatisticsWithN2Test() - { - var summary = new Statistics(1, 2); - Print(summary); - Assert.Equal(1, summary.Min); - Assert.Equal(0.5, summary.LowerFence); - Assert.Equal(1.25, summary.Q1); - Assert.Equal(1.5, summary.Median); - Assert.Equal(1.5, summary.Mean); - Assert.Equal(1.75, summary.Q3); - Assert.Equal(2.5, summary.UpperFence); - Assert.Equal(2, summary.Max); - Assert.Equal(0.5, summary.InterquartileRange); - Assert.Equal(0.70711, summary.StandardDeviation, 4); - Assert.Equal(Array.Empty(), summary.AllOutliers); - Assert.Equal(1, summary.Percentiles.P0); - Assert.Equal(1.25, summary.Percentiles.P25); - Assert.Equal(1.5, summary.Percentiles.P50); - Assert.Equal(1.85, summary.Percentiles.P85); - Assert.Equal(1.95, summary.Percentiles.P95); - Assert.Equal(2, summary.Percentiles.P100); - } + [Fact] + public void StatisticsWithN7Test() + { + var summary = new Statistics(1, 2, 4, 8, 16, 32, 64); + Print(summary); + AssertEqual(1, summary.Min); + AssertEqual(-28.5, summary.LowerFence); + AssertEqual(3, summary.Q1); + AssertEqual(8, summary.Median); + AssertEqual(18.1428571429, summary.Mean, AbsoluteEqualityComparer.E5); + AssertEqual(24, summary.Q3); + AssertEqual(55.5, summary.UpperFence); + AssertEqual(64, summary.Max); + AssertEqual(21, summary.InterquartileRange); + AssertEqual(22.9378, summary.StandardDeviation, AbsoluteEqualityComparer.E4); + AssertEqual(new[] { 64.0 }, summary.AllOutliers); + AssertEqual(1, summary.Percentiles.P0); + AssertEqual(3, summary.Percentiles.P25); + AssertEqual(8, summary.Percentiles.P50); + AssertEqual(35.2, summary.Percentiles.P85, AbsoluteEqualityComparer.E4); + AssertEqual(54.4, summary.Percentiles.P95, AbsoluteEqualityComparer.E4); + AssertEqual(64, summary.Percentiles.P100); + } - [Fact] - public void StatisticsWithN3Test() - { - var summary = new Statistics(1, 2, 4); - Print(summary); - Assert.Equal(1, summary.Min); - Assert.Equal(-0.75, summary.LowerFence); - Assert.Equal(1.5, summary.Q1); - Assert.Equal(2, summary.Median); - Assert.Equal(2.333333, summary.Mean, 5); - Assert.Equal(3, summary.Q3); - Assert.Equal(5.25, summary.UpperFence); - Assert.Equal(4, summary.Max); - Assert.Equal(1.5, summary.InterquartileRange); - Assert.Equal(1.52753, summary.StandardDeviation, 4); - Assert.Equal(Array.Empty(), summary.AllOutliers); - Assert.Equal(1, summary.Percentiles.P0); - Assert.Equal(1.5, summary.Percentiles.P25); - Assert.Equal(2, summary.Percentiles.P50); - Assert.Equal(3.4, summary.Percentiles.P85); - Assert.Equal(3.8, summary.Percentiles.P95); - Assert.Equal(4, summary.Percentiles.P100); - } + [Fact] + public void OutlierTest() + { + var summary = new Statistics(1, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 10, 10.1); + Print(summary); + AssertEqual(new[] { 10, 10.1 }, summary.AllOutliers); + AssertEqual(new[] { 10, 10.1 }, summary.UpperOutliers); + AssertEqual(Array.Empty(), summary.LowerOutliers); + } - [Fact] - public void StatisticsWithN7Test() - { - var summary = new Statistics(1, 2, 4, 8, 16, 32, 64); - Print(summary); - Assert.Equal(1, summary.Min); - Assert.Equal(-28.5, summary.LowerFence); - Assert.Equal(3, summary.Q1); - Assert.Equal(8, summary.Median); - Assert.Equal(18.1428571429, summary.Mean, 5); - Assert.Equal(24, summary.Q3); - Assert.Equal(55.5, summary.UpperFence); - Assert.Equal(64, summary.Max); - Assert.Equal(21, summary.InterquartileRange); - Assert.Equal(22.9378, summary.StandardDeviation, 4); - Assert.Equal(new[] { 64.0 }, summary.AllOutliers); - Assert.Equal(1, summary.Percentiles.P0); - Assert.Equal(3, summary.Percentiles.P25); - Assert.Equal(8, summary.Percentiles.P50); - Assert.Equal(35.2, summary.Percentiles.P85, 4); - Assert.Equal(54.4, summary.Percentiles.P95, 4); - Assert.Equal(64, summary.Percentiles.P100); - } + [Fact] + public void ConfidenceIntervalTest() + { + var summary = new Statistics(Enumerable.Range(1, 30)); + Print(summary); + Assert.Equal("99.9%", summary.PerfolizerConfidenceInterval.ConfidenceLevel.ToString()); + AssertEqual(15.5, summary.PerfolizerConfidenceInterval.Estimation); + AssertEqual(9.618329, summary.PerfolizerConfidenceInterval.Lower, AbsoluteEqualityComparer.E4); + AssertEqual(21.38167, summary.PerfolizerConfidenceInterval.Upper, AbsoluteEqualityComparer.E4); + } - [Fact] - public void OutlierTest() - { - var summary = new Statistics(1, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 10, 10.1); - Print(summary); - Assert.Equal(new[] { 10, 10.1 }, summary.AllOutliers); - Assert.Equal(new[] { 10, 10.1 }, summary.UpperOutliers); - Assert.Equal(Array.Empty(), summary.LowerOutliers); - } + [Fact] + public void PercentileValuesWithN30Test() + { + var summary = new Statistics(Enumerable.Range(1, 30)); + Print(summary); + AssertEqual(1, summary.Percentiles.P0); + AssertEqual(8.25, summary.Percentiles.P25); + AssertEqual(15.5, summary.Percentiles.P50); + AssertEqual(20.43, summary.Percentiles.P67, AbsoluteEqualityComparer.E4); + AssertEqual(24.2, summary.Percentiles.P80, AbsoluteEqualityComparer.E4); + AssertEqual(25.65, summary.Percentiles.P85); + AssertEqual(27.1, summary.Percentiles.P90); + AssertEqual(28.55, summary.Percentiles.P95, AbsoluteEqualityComparer.E4); + AssertEqual(30, summary.Percentiles.P100); + } - [Fact] - public void ConfidenceIntervalTest() - { - var summary = new Statistics(Enumerable.Range(1, 30)); - Print(summary); - Assert.Equal("99.9%", summary.ConfidenceInterval.Level.ToPercentStr()); - Assert.Equal(15.5, summary.ConfidenceInterval.Mean); - Assert.Equal(summary.StandardError, summary.ConfidenceInterval.StandardError); - Assert.Equal(9.618329, summary.ConfidenceInterval.Lower, 4); - Assert.Equal(21.38167, summary.ConfidenceInterval.Upper, 4); - } + [Fact] + public void PercentileValuesWithN90Test() + { + var a = Enumerable.Range(1, 30); + var b = Enumerable.Repeat(0, 30).Concat(a); + var c = b.Concat(Enumerable.Repeat(31, 30)); + var summary = new Statistics(c); + Print(summary); + AssertEqual(0, summary.Percentiles.P0); + AssertEqual(0, summary.Percentiles.P25); + AssertEqual(15.5, summary.Percentiles.P50); + AssertEqual(30.63, summary.Percentiles.P67, AbsoluteEqualityComparer.E4); + AssertEqual(31, summary.Percentiles.P80); + AssertEqual(31, summary.Percentiles.P85); + AssertEqual(31, summary.Percentiles.P90); + AssertEqual(31, summary.Percentiles.P95); + AssertEqual(31, summary.Percentiles.P100); + } - [Fact] - public void PercentileValuesWithN30Test() - { - var summary = new Statistics(Enumerable.Range(1, 30)); - Print(summary); - Assert.Equal(1, summary.Percentiles.P0); - Assert.Equal(8.25, summary.Percentiles.P25); - Assert.Equal(15.5, summary.Percentiles.P50); - Assert.Equal(20.43, summary.Percentiles.P67, 4); - Assert.Equal(24.2, summary.Percentiles.P80, 4); - Assert.Equal(25.65, summary.Percentiles.P85); - Assert.Equal(27.1, summary.Percentiles.P90); - Assert.Equal(28.55, summary.Percentiles.P95, 4); - Assert.Equal(30, summary.Percentiles.P100); - } + [AssertionMethod] + private static void AssertEqual(double a, double b, IEqualityComparer? comparer = null) + { + Assert.Equal(a, b, comparer ?? AbsoluteEqualityComparer.E9); + } - [Fact] - public void PercentileValuesWithN90Test() - { - var a = Enumerable.Range(1, 30); - var b = Enumerable.Repeat(0, 30).Concat(a); - var c = b.Concat(Enumerable.Repeat(31, 30)); - var summary = new Statistics(c); - Print(summary); - Assert.Equal(0, summary.Percentiles.P0); - Assert.Equal(0, summary.Percentiles.P25); - Assert.Equal(15.5, summary.Percentiles.P50); - Assert.Equal(30.63, summary.Percentiles.P67, 4); - Assert.Equal(31, summary.Percentiles.P80); - Assert.Equal(31, summary.Percentiles.P85); - Assert.Equal(31, summary.Percentiles.P90); - Assert.Equal(31, summary.Percentiles.P95); - Assert.Equal(31, summary.Percentiles.P100); - } + [AssertionMethod] + private static void AssertEqual(double[] a, double[] b, IEqualityComparer? comparer = null) + { + Assert.Equal(a, b, comparer ?? AbsoluteEqualityComparer.E9); } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs index 6b1fad64a6..7b17baa098 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Jobs; @@ -56,6 +57,19 @@ public Measurement RunIteration(IterationData data) public RunResults Run() => default; + internal (long invokeCount, List measurements) Run(EngineStage stage, long invokeCount = 0) + { + var measurements = stage.GetMeasurementList(); + int iterationIndex = 1; + while (stage.GetShouldRunIteration(measurements, ref invokeCount)) + { + var measurement = RunIteration(new IterationData(stage.Mode, stage.Stage, iterationIndex, invokeCount, 1)); + measurements.Add(measurement); + ++iterationIndex; + } + return (invokeCount, measurements); + } + public void WriteLine() => output.WriteLine(""); public void WriteLine(string line) => output.WriteLine(line); diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs index ce54b83bbb..bbe715d660 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs @@ -2,9 +2,11 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using System.Reflection; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Disassemblers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Reports; @@ -14,18 +16,42 @@ using BenchmarkDotNet.Toolchains.Results; using BenchmarkDotNet.Validators; using Perfolizer.Horology; +using Perfolizer.Metrology; using static BenchmarkDotNet.Reports.SummaryTable.SummaryTableColumn; +using Measurement = BenchmarkDotNet.Reports.Measurement; +using MethodInfo = System.Reflection.MethodInfo; namespace BenchmarkDotNet.Tests.Mocks { public static class MockFactory { - public static Summary CreateSummary(Type benchmarkType) + public static Summary CreateSummary(Type benchmarkType, params IColumnHidingRule[] columHidingRules) { var runInfo = BenchmarkConverter.TypeToBenchmarks(benchmarkType); return new Summary( "MockSummary", - runInfo.BenchmarksCases.Select((benchmark, index) => CreateReport(benchmark, 5, (index + 1) * 100)).ToImmutableArray(), + runInfo.BenchmarksCases.Select((benchmark, index) => CreateReport(benchmark, 30, (index + 1) * 100)).ToImmutableArray(), + new HostEnvironmentInfoBuilder().WithoutDotNetSdkVersion().Build(), + string.Empty, + string.Empty, + TimeSpan.FromMinutes(1), + TestCultureInfo.Instance, + ImmutableArray.Empty, + ImmutableArray.Create(columHidingRules)); + } + + public static Summary CreateSummaryWithBiasedDistribution(Type benchmarkType, int min, int median, int max, int n) + { + var runInfo = BenchmarkConverter.TypeToBenchmarks(benchmarkType); + return new Summary( + $"MockSummary-N{n}", + runInfo.BenchmarksCases.Select((benchmark, index) => CreateReportWithBiasedDistribution( + benchmark, + (index + 1) * min, + (index + 1) * median, + (index + 1) * max, + n, + Array.Empty())).ToImmutableArray(), new HostEnvironmentInfoBuilder().WithoutDotNetSdkVersion().Build(), string.Empty, string.Empty, @@ -74,7 +100,7 @@ public static Summary CreateSummary(IConfig config, bool hugeSd, Metric[] metric ImmutableArray.Empty); public static SummaryStyle CreateSummaryStyle(bool printUnitsInHeader = false, bool printUnitsInContent = true, bool printZeroValuesInContent = false, - SizeUnit sizeUnit = null, TimeUnit timeUnit = null, TextJustification textColumnJustification = TextJustification.Left, + SizeUnit? sizeUnit = null, TimeUnit? timeUnit = null, TextJustification textColumnJustification = TextJustification.Left, TextJustification numericColumnJustification = TextJustification.Left) => new SummaryStyle(DefaultCultureInfo.Instance, printUnitsInHeader, sizeUnit, timeUnit, printUnitsInContent: printUnitsInContent, printZeroValuesInContent: printZeroValuesInContent, textColumnJustification: textColumnJustification, @@ -116,6 +142,40 @@ private static BenchmarkReport CreateReport(BenchmarkCase benchmarkCase, bool hu return new BenchmarkReport(true, benchmarkCase, buildResult, buildResult, new List { executeResult }, metrics); } + private static BenchmarkReport CreateReportWithBiasedDistribution(BenchmarkCase benchmarkCase, int min, int median, int max, int n, Metric[] metrics) + { + var buildResult = BuildResult.Success(GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty())); + bool isFoo = benchmarkCase.Descriptor.WorkloadMethodDisplayInfo == "Foo"; + bool isBar = benchmarkCase.Descriptor.WorkloadMethodDisplayInfo == "Bar"; + var measurements = from i in Enumerable.Range(0, Math.Max(1, n / 9)) + from m in isFoo ? new[] + { + new Measurement(1, IterationMode.Workload, IterationStage.Result, 1, 1, min), // 1 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 2, 1, min + ((median - min) / 2) + 1), // 3 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 4, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 7, 1, median + ((max - median) / 2)), // 7 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 8, 1, median + ((max - median) / 2)), // 7 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 9, 1, max), // 10 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 9, 1, max), // 10 + } : new[] + { + new Measurement(1, IterationMode.Workload, IterationStage.Result, 1, 1, min), // 1 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 1, 1, min), // 1 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 2, 1, min + ((median - min) / 2) + 1), // 3 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 2, 1, min + ((median - min) / 2) + 1), // 3 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 4, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 7, 1, median + ((max - median) / 2)), // 7 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 9, 1, max), // 10 + } + select m; + var executeResult = new ExecuteResult(measurements.Take(n).ToList(), default, default, 0); + return new BenchmarkReport(true, benchmarkCase, buildResult, buildResult, new List { executeResult }, metrics); + } + [LongRunJob] public class MockBenchmarkClass { @@ -123,5 +183,10 @@ [Benchmark] public void Foo() { } [Benchmark] public void Bar() { } } + + public static readonly Type MockType = typeof(MockBenchmarkClass); + + public static readonly MethodInfo MockMethodInfo = MockType.GetTypeInfo().GetMethods() + .Single(method => method.Name == nameof(MockBenchmarkClass.Foo)); } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Mocks/Toolchain/MockToolchain.cs b/tests/BenchmarkDotNet.Tests/Mocks/Toolchain/MockToolchain.cs index c25929854e..365a55cb88 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/Toolchain/MockToolchain.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/Toolchain/MockToolchain.cs @@ -43,7 +43,7 @@ private class MockExecutor : IExecutor public MockExecutor(Func> measurer) => this.measurer = measurer; - public ExecuteResult Execute(ExecuteParameters executeParameters) => new (measurer(executeParameters.BenchmarkCase)); + public ExecuteResult Execute(ExecuteParameters executeParameters) => new(measurer(executeParameters.BenchmarkCase)); } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs b/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs index 04cff38e30..38b9167e99 100644 --- a/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs +++ b/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs @@ -11,6 +11,7 @@ using BenchmarkDotNet.Parameters; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Mocks; using BenchmarkDotNet.Validators; using Xunit; @@ -18,11 +19,11 @@ namespace BenchmarkDotNet.Tests.Order { public class DefaultOrdererTests { - private static Summary CreateMockSummary() => new ("", ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), + private static Summary CreateMockSummary() => new("", ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), "", "", TimeSpan.Zero, CultureInfo.InvariantCulture, ImmutableArray.Empty, ImmutableArray.Empty); - private static BenchmarkCase CreateBenchmarkCase(string category, int parameter, params BenchmarkLogicalGroupRule[] rules) => new ( - new Descriptor(null, null, categories: new[] { category }), + private static BenchmarkCase CreateBenchmarkCase(string category, int parameter, params BenchmarkLogicalGroupRule[] rules) => new( + new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo, categories: new[] { category }), new Job(), new ParameterInstances(new[] { diff --git a/tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs b/tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs new file mode 100644 index 0000000000..c0e645ea22 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs @@ -0,0 +1,146 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.CsProj; +using System.Linq; +using Xunit; + +namespace BenchmarkDotNet.Tests.Order; + +public class JobOrderTests +{ + [Fact] + public void TestJobOrders_ByJobId() + { + // Arrange + Job[] jobs = + [ + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80) + .WithRuntime(CoreRuntime.Core80) + .WithId("v1.4.1"), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90) + .WithRuntime(CoreRuntime.Core90) + .WithId("v1.4.10"), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0) + .WithRuntime(CoreRuntime.Core10_0) + .WithId("v1.4.2"), + ]; + + // Verify jobs are sorted by JobId's ordinal order. + { + // Act + var comparer = JobComparer.Instance; + var results = jobs.OrderBy(x => x, comparer) + .Select(x => x.Job.Id) + .ToArray(); + + // Assert + Assert.Equal(["v1.4.1", "v1.4.10", "v1.4.2"], results); + } + + // Verify jobs are sorted by JobId's numeric order. + { + // Act + var comparer = JobComparer.Numeric; + var results = jobs.OrderBy(d => d, comparer) + .Select(x => x.Job.Id) + .ToArray(); + // Assert + Assert.Equal(["v1.4.1", "v1.4.2", "v1.4.10"], results); + } + } + + [Fact] + public void TestJobOrders_ByRuntime() + { + // Arrange + Job[] jobs = + [ + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0) + .WithRuntime(CoreRuntime.Core80), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90) + .WithRuntime(CoreRuntime.Core90), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80) + .WithRuntime(CoreRuntime.Core10_0), + ]; + + // Act + // Verify jobs are sorted by Runtime's numeric order. + var results = jobs.OrderBy(d => d, JobComparer.Numeric) + .Select(x => x.Job.Environment.GetRuntime().Name) + .ToArray(); + + // Assert + var expected = new[] + { + CoreRuntime.Core80.Name, + CoreRuntime.Core90.Name, + CoreRuntime.Core10_0.Name + }; + Assert.Equal(expected, results); + } + + [Fact] + public void TestJobOrders_ByToolchain() + { + // Arrange + Job[] jobs = + [ + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80), + ]; + + // Act + // Verify jobs are sorted by Toolchain's numeric order. + var results = jobs.OrderBy(d => d, JobComparer.Numeric) + .Select(x => x.Job.GetToolchain().Name) + .ToArray(); + + // Assert + var expected = new[] + { + CsProjCoreToolchain.NetCoreApp80.Name, + CsProjCoreToolchain.NetCoreApp90.Name, + CsProjCoreToolchain.NetCoreApp10_0.Name, + }; + Assert.Equal(expected, results); + } + + [Theory] + [InlineData("item1", "item1", 0)] + [InlineData("item123", "item123", 0)] + // Compare different values + [InlineData("item1", "item2", -1)] + [InlineData("item2", "item1", 1)] + [InlineData("item2", "item10", -1)] + [InlineData("item10", "item2", 1)] + [InlineData("item1a", "item1b", -1)] + [InlineData("item1b", "item1a", 1)] + [InlineData("item", "item1", -1)] + [InlineData("item10", "item", 1)] + [InlineData(".NET 8", ".NET 10", -1)] + [InlineData(".NET 10", ".NET 8", 1)] + [InlineData("v1.4.1", "v1.4.10", -1)] + [InlineData("v1.4.10", "v1.4.2", 1)] + // Compare zero paddeed numeric string. + [InlineData("item01", "item1", 0)] + [InlineData("item001", "item1", 0)] + [InlineData("item1", "item001", 0)] + [InlineData("item1", "item01", 0)] + [InlineData("item9", "item09", 0)] + [InlineData(".NET 08", ".NET 10", -1)] + [InlineData(".NET 10", ".NET 08", 1)] + // Arguments that contains null + [InlineData(null, "a", -1)] + [InlineData("a", null, 1)] + [InlineData(null, null, 0)] + public void TestNumericComparer(string? a, string? b, int expectedSign) + { + int result = new JobComparer.NumericStringComparer().Compare(a, b); + Assert.Equal(expectedSign, NormalizeSign(result)); + + static int NormalizeSign(int value) + => value == 0 ? 0 : value < 0 ? -1 : 1; + } +} diff --git a/tests/BenchmarkDotNet.Tests/ParamsSourceTests.cs b/tests/BenchmarkDotNet.Tests/ParamsSourceTests.cs index 7ee8695475..c32b48d721 100644 --- a/tests/BenchmarkDotNet.Tests/ParamsSourceTests.cs +++ b/tests/BenchmarkDotNet.Tests/ParamsSourceTests.cs @@ -21,7 +21,7 @@ public static IEnumerable Values() { yield return null; yield return ValueTuple.Create(10); - yield return ValueTuple.Create(10, 20); + yield return (10, 20); yield return (10, 20, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); } diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/Infra/PerfonarMock.cs b/tests/BenchmarkDotNet.Tests/Perfonar/Infra/PerfonarMock.cs new file mode 100644 index 0000000000..f63caeafa4 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/Infra/PerfonarMock.cs @@ -0,0 +1,13 @@ +using BenchmarkDotNet.Properties; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Tests.Perfonar.Infra; + +public static class PerfonarMock +{ + public static readonly EngineInfo Engine = new() + { + Name = BenchmarkDotNetInfo.BenchmarkDotNetCaption, + Version = "0.1729.0-mock" + }; +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/Infra/PerfonarTestExtensions.cs b/tests/BenchmarkDotNet.Tests/Perfonar/Infra/PerfonarTestExtensions.cs new file mode 100644 index 0000000000..3df26943cf --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/Infra/PerfonarTestExtensions.cs @@ -0,0 +1,22 @@ +using Perfolizer.Models; +using Perfolizer.Metrology; + +namespace BenchmarkDotNet.Tests.Perfonar.Infra; + +public static class PerfonarTestExtensions +{ + public static EntryInfo AddMetrics(this EntryInfo entry, params string[] metrics) + { + for (int i = 0; i < metrics.Length; i++) + { + var measurement = Measurement.Parse(metrics[i]); + entry.Add(new EntryInfo + { + IterationIndex = i, + Value = measurement.NominalValue, + Unit = measurement.Unit + }); + } + return entry; + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs b/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs new file mode 100644 index 0000000000..81869534a1 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs @@ -0,0 +1,174 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Models; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Tests.Builders; +using BenchmarkDotNet.Tests.Infra; +using BenchmarkDotNet.Tests.Perfonar.Infra; +using JetBrains.Annotations; +using Perfolizer.Models; +using Perfolizer.Json; +using Perfolizer.Perfonar.Base; +using Perfolizer.Perfonar.Tables; +using VerifyXunit; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Perfonar; + +public class PerfonarTests(ITestOutputHelper output) +{ + [Theory] + [MemberData(nameof(EntryDataKeys))] + public Task PerfonarIndexTest(string key) => VerifyString(key, new PerfonarIndex(TableDataMap[key].RootEntry).Dump()); + + [Theory] + [MemberData(nameof(EntryDataKeys))] + public Task PerfonarTableTest(string key) + { + var table = TableDataMap[key]; + string markdown = table.ToMarkdown(new PerfonarTableStyle()); + string json = LightJsonSerializer.Serialize(table.RootEntry, new LightJsonSettings { Indent = true }); + return VerifyString(key, markdown + "\n" + json); + } + + private static readonly IDictionary TableDataMap = new Dictionary + { + { + "default01", new PerfonarTable( + Root().Add( + Benchmark("Foo", "10ns", "11ns", "12ns"), + Benchmark("Bar", "200ns", "201ns", "202ns") + ), + GetDefaultTableConfig() + ) + }, + { + "default02", new PerfonarTable( + Root().Add( + Job(RuntimeMoniker.Net481).Add( + Benchmark("Foo", "10ns", "11ns", "12ns"), + Benchmark("Bar", "20ns", "21ns", "22ns")), + Job(RuntimeMoniker.Net70).Add( + Benchmark("Foo", "30ns", "31ns", "32ns"), + Benchmark("Bar", "40ns", "41ns", "42ns"))), + GetDefaultTableConfig() + ) + }, + { + "default03", new PerfonarTable( + Root().Add( + Job(RuntimeMoniker.Net70, Jit.RyuJit).Add( + Benchmark("Foo", "30ns", "31ns", "32ns"), + Benchmark("Bar", "40ns", "41ns", "42ns"))), + GetDefaultTableConfig() + ) + }, + { + "default04", new PerfonarTable( + Root().Add( + Job(RuntimeMoniker.Net481, Jit.LegacyJit).Add( + Benchmark("Foo", "10ns", "11ns", "12ns"), + Benchmark("Bar", "20ns", "21ns", "22ns")), + Job(RuntimeMoniker.Net70, Jit.RyuJit).Add( + Benchmark("Foo", "30ns", "31ns", "32ns"), + Benchmark("Bar", "40ns", "41ns", "42ns"))), + GetDefaultTableConfig() + ) + }, + { + "default05", new PerfonarTable( + Root().Add( + Enumerable.Range(0, 20).Select(index => + Job((RuntimeMoniker)index, Jit.RyuJit, index).Add( + Benchmark("Foo", index * 10 + 1 + "ns", index * 10 + 2 + "ns", index * 10 + 3 + "ns"), + Benchmark("Bar", index * 10 + 6 + "ns", index * 10 + 7 + "ns", index * 10 + 8 + "ns") + )).ToArray()), + GetDefaultTableConfig() + ) + }, + { + "sort01", new PerfonarTable( + new EntryInfo().Add(Benchmark("Foo", "10ms"), Benchmark("Bar", "20ms")), + new PerfonarTableConfig + { + ColumnDefinitions = + { + new PerfonarColumnDefinition(".benchmark.method"), + new PerfonarColumnDefinition("=center") + }, + SortPolicies = + [ + new PerfonarSortPolicy("=center", PerfonarSortDirection.Descending) + ] + }) + }, + { + "params01", new PerfonarTable( + new EntryInfo().Add( + new EntryInfo + { + Parameters = new Dictionary { { "A", 1 }, { "B", 2 } } + }.Add(Benchmark("Foo", "10ms"))).Add( + new EntryInfo + { + Parameters = new Dictionary { { "A", 10 }, { "B", 20 } } + }.Add(Benchmark("Bar", "20ms"))), + new PerfonarTableConfig + { + ColumnDefinitions = + { + new PerfonarColumnDefinition(".benchmark.method"), + new PerfonarColumnDefinition(".parameters"), + new PerfonarColumnDefinition("=center") + } + } + ) + } + }; + + [UsedImplicitly] public static TheoryData EntryDataKeys = TheoryDataHelper.Create(TableDataMap.Keys); + + private static EntryInfo Root() => new EntryInfo + { + Engine = PerfonarMock.Engine, + Host = new HostEnvironmentInfoBuilder().Build().ToPerfonar() + }; + + private static PerfonarTableConfig GetDefaultTableConfig() => new() + { + ColumnDefinitions = + [ + new PerfonarColumnDefinition(".engine") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PerfonarColumnDefinition(".host.os") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PerfonarColumnDefinition(".host.cpu") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PerfonarColumnDefinition(".benchmark") { Cloud = "secondary" }, + new PerfonarColumnDefinition(".job") { Cloud = "secondary", Compressed = true }, + new PerfonarColumnDefinition("=center"), + new PerfonarColumnDefinition("=spread") + ] + }; + + private static EntryInfo Job(RuntimeMoniker? runtime = null, Jit? jit = null, int? affinity = null) => new EntryInfo + { + Job = new JobInfo + { + Environment = new BdnEnvironment { Runtime = runtime, Jit = jit, Affinity = affinity } + } + }; + + private static EntryInfo Benchmark(string name, params string[] metrics) => new EntryInfo + { + Benchmark = new BdnBenchmark { Type = "Bench", Method = name } + }.AddMetrics(metrics); + + private Task VerifyString(string key, string content) + { + output.WriteLine(content); + var settings = VerifyHelper.Create("Perfonar"); + settings.UseParameters(key); + return Verifier.Verify(content, settings); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default01.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default01.verified.txt new file mode 100644 index 0000000000..a412c88898 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default01.verified.txt @@ -0,0 +1,190 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .engine + .engine.name + .engine.version + .host + .host.chronometerFrequency + .host.configuration + .host.cpu + .host.cpu.logicalCoreCount + .host.cpu.maxFrequencyHz + .host.cpu.nominalFrequencyHz + .host.cpu.physicalCoreCount + .host.cpu.physicalProcessorCount + .host.cpu.processorName + .host.dotNetSdkVersion + .host.hardwareTimerKind + .host.hasAttachedDebugger + .host.hasRyuJit + .host.os + .host.os.display + .host.runtimeVersion + .iterationIndex + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .unit = ns + .value = 10 +[Entry1] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .unit = ns + .value = 11 +[Entry2] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .unit = ns + .value = 12 +[Entry3] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .unit = ns + .value = 200 +[Entry4] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .unit = ns + .value = 201 +[Entry5] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .unit = ns + .value = 202 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default02.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default02.verified.txt new file mode 100644 index 0000000000..c8759facf6 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default02.verified.txt @@ -0,0 +1,391 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .engine + .engine.name + .engine.version + .host + .host.chronometerFrequency + .host.configuration + .host.cpu + .host.cpu.logicalCoreCount + .host.cpu.maxFrequencyHz + .host.cpu.nominalFrequencyHz + .host.cpu.physicalCoreCount + .host.cpu.physicalProcessorCount + .host.cpu.processorName + .host.dotNetSdkVersion + .host.hardwareTimerKind + .host.hasAttachedDebugger + .host.hasRyuJit + .host.os + .host.os.display + .host.runtimeVersion + .iterationIndex + .job + .job.environment + .job.environment.runtime + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 10 +[Entry1] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 11 +[Entry2] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 12 +[Entry3] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 20 +[Entry4] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 21 +[Entry5] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 22 +[Entry6] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 30 +[Entry7] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 31 +[Entry8] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 32 +[Entry9] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 40 +[Entry10] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 41 +[Entry11] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 42 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default03.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default03.verified.txt new file mode 100644 index 0000000000..af8395637c --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default03.verified.txt @@ -0,0 +1,218 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .engine + .engine.name + .engine.version + .host + .host.chronometerFrequency + .host.configuration + .host.cpu + .host.cpu.logicalCoreCount + .host.cpu.maxFrequencyHz + .host.cpu.nominalFrequencyHz + .host.cpu.physicalCoreCount + .host.cpu.physicalProcessorCount + .host.cpu.processorName + .host.dotNetSdkVersion + .host.hardwareTimerKind + .host.hasAttachedDebugger + .host.hasRyuJit + .host.os + .host.os.display + .host.runtimeVersion + .iterationIndex + .job + .job.environment + .job.environment.jit + .job.environment.runtime + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 30 +[Entry1] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 31 +[Entry2] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 32 +[Entry3] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 40 +[Entry4] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 41 +[Entry5] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 42 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default04.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default04.verified.txt new file mode 100644 index 0000000000..1179e76cd5 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default04.verified.txt @@ -0,0 +1,404 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .engine + .engine.name + .engine.version + .host + .host.chronometerFrequency + .host.configuration + .host.cpu + .host.cpu.logicalCoreCount + .host.cpu.maxFrequencyHz + .host.cpu.nominalFrequencyHz + .host.cpu.physicalCoreCount + .host.cpu.physicalProcessorCount + .host.cpu.processorName + .host.dotNetSdkVersion + .host.hardwareTimerKind + .host.hasAttachedDebugger + .host.hasRyuJit + .host.os + .host.os.display + .host.runtimeVersion + .iterationIndex + .job + .job.environment + .job.environment.jit + .job.environment.runtime + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 10 +[Entry1] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 11 +[Entry2] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 12 +[Entry3] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 20 +[Entry4] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 21 +[Entry5] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 22 +[Entry6] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 30 +[Entry7] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 31 +[Entry8] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 32 +[Entry9] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 40 +[Entry10] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 41 +[Entry11] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 42 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default05.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default05.verified.txt new file mode 100644 index 0000000000..ef0556ab95 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default05.verified.txt @@ -0,0 +1,3873 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .engine + .engine.name + .engine.version + .host + .host.chronometerFrequency + .host.configuration + .host.cpu + .host.cpu.logicalCoreCount + .host.cpu.maxFrequencyHz + .host.cpu.nominalFrequencyHz + .host.cpu.physicalCoreCount + .host.cpu.physicalProcessorCount + .host.cpu.processorName + .host.dotNetSdkVersion + .host.hardwareTimerKind + .host.hasAttachedDebugger + .host.hasRyuJit + .host.os + .host.os.display + .host.runtimeVersion + .iterationIndex + .job + .job.environment + .job.environment.affinity + .job.environment.jit + .job.environment.runtime + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 1 +[Entry1] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 2 +[Entry2] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 3 +[Entry3] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 6 +[Entry4] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 7 +[Entry5] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 8 +[Entry6] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 11 +[Entry7] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 12 +[Entry8] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 13 +[Entry9] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 16 +[Entry10] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 17 +[Entry11] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 18 +[Entry12] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 21 +[Entry13] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 22 +[Entry14] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 23 +[Entry15] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 26 +[Entry16] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 27 +[Entry17] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 28 +[Entry18] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 31 +[Entry19] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 32 +[Entry20] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 33 +[Entry21] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 36 +[Entry22] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 37 +[Entry23] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 38 +[Entry24] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 41 +[Entry25] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 42 +[Entry26] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 43 +[Entry27] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 46 +[Entry28] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 47 +[Entry29] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 48 +[Entry30] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 51 +[Entry31] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 52 +[Entry32] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 53 +[Entry33] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 56 +[Entry34] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 57 +[Entry35] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 58 +[Entry36] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 61 +[Entry37] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 62 +[Entry38] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 63 +[Entry39] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 66 +[Entry40] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 67 +[Entry41] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 68 +[Entry42] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 71 +[Entry43] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 72 +[Entry44] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 73 +[Entry45] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 76 +[Entry46] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 77 +[Entry47] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 78 +[Entry48] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 81 +[Entry49] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 82 +[Entry50] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 83 +[Entry51] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 86 +[Entry52] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 87 +[Entry53] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 88 +[Entry54] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 91 +[Entry55] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 92 +[Entry56] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 93 +[Entry57] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 96 +[Entry58] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 97 +[Entry59] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 98 +[Entry60] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 101 +[Entry61] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 102 +[Entry62] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 103 +[Entry63] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 106 +[Entry64] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 107 +[Entry65] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 108 +[Entry66] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 111 +[Entry67] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 112 +[Entry68] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 113 +[Entry69] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 116 +[Entry70] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 117 +[Entry71] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 118 +[Entry72] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 121 +[Entry73] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 122 +[Entry74] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 123 +[Entry75] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 126 +[Entry76] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 127 +[Entry77] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 128 +[Entry78] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 131 +[Entry79] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 132 +[Entry80] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 133 +[Entry81] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 136 +[Entry82] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 137 +[Entry83] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 138 +[Entry84] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 141 +[Entry85] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 142 +[Entry86] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 143 +[Entry87] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 146 +[Entry88] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 147 +[Entry89] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 148 +[Entry90] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 151 +[Entry91] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 152 +[Entry92] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 153 +[Entry93] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 156 +[Entry94] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 157 +[Entry95] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 158 +[Entry96] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 161 +[Entry97] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 162 +[Entry98] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 163 +[Entry99] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 166 +[Entry100] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 167 +[Entry101] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 168 +[Entry102] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 171 +[Entry103] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 172 +[Entry104] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 173 +[Entry105] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 176 +[Entry106] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 177 +[Entry107] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 178 +[Entry108] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 181 +[Entry109] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 182 +[Entry110] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 183 +[Entry111] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 186 +[Entry112] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 187 +[Entry113] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 188 +[Entry114] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 191 +[Entry115] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 192 +[Entry116] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 193 +[Entry117] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 196 +[Entry118] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 197 +[Entry119] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 198 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=params01.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=params01.verified.txt new file mode 100644 index 0000000000..ccc03f6896 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=params01.verified.txt @@ -0,0 +1,31 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .iterationIndex + .parameters + .parameters.a + .parameters.b + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .iterationIndex = 0 + .parameters = + .parameters.a = 1 + .parameters.b = 2 + .unit = ms + .value = 10 +[Entry1] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .iterationIndex = 0 + .parameters = + .parameters.a = 10 + .parameters.b = 20 + .unit = ms + .value = 20 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=sort01.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=sort01.verified.txt new file mode 100644 index 0000000000..f1c1835a59 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=sort01.verified.txt @@ -0,0 +1,22 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .iterationIndex + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .iterationIndex = 0 + .unit = ms + .value = 10 +[Entry1] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .iterationIndex = 0 + .unit = ms + .value = 20 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default01.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default01.verified.txt new file mode 100644 index 0000000000..9fe1919a4e --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default01.verified.txt @@ -0,0 +1,84 @@ +BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + +Type = Bench + +| Method | Center | Spread | +|:-------|---------:|--------:| +| Foo | 11.0 ns | 0.81 ns | +| Bar | 201.0 ns | 0.81 ns | + +{ + "engine": { + "name": "BenchmarkDotNet", + "version": "0.1729.0-mock" + }, + "host": { + "runtimeVersion": "Clr 4.0.x.mock", + "hasAttachedDebugger": false, + "hasRyuJit": true, + "configuration": "CONFIGURATION", + "dotNetSdkVersion": "1.0.x.mock", + "chronometerFrequency": 2531248, + "hardwareTimerKind": "Tsc", + "os": { + "display": "Microsoft Windows NT 10.0.x.mock" + }, + "cpu": { + "processorName": "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + "physicalProcessorCount": 1, + "physicalCoreCount": 4, + "logicalCoreCount": 8, + "nominalFrequencyHz": 3100000000, + "maxFrequencyHz": 3100000000 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 10, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 11, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 12, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 200, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 201, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 202, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default02.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default02.verified.txt new file mode 100644 index 0000000000..a3ade54aea --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default02.verified.txt @@ -0,0 +1,150 @@ +BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + +Type = Bench + +| Method | Runtime | Center | Spread | +|:-------|:--------|--------:|--------:| +| Foo | Net481 | 11.0 ns | 0.81 ns | +| Bar | Net481 | 21.0 ns | 0.81 ns | +| Foo | Net70 | 31.0 ns | 0.81 ns | +| Bar | Net70 | 41.0 ns | 0.81 ns | + +{ + "engine": { + "name": "BenchmarkDotNet", + "version": "0.1729.0-mock" + }, + "host": { + "runtimeVersion": "Clr 4.0.x.mock", + "hasAttachedDebugger": false, + "hasRyuJit": true, + "configuration": "CONFIGURATION", + "dotNetSdkVersion": "1.0.x.mock", + "chronometerFrequency": 2531248, + "hardwareTimerKind": "Tsc", + "os": { + "display": "Microsoft Windows NT 10.0.x.mock" + }, + "cpu": { + "processorName": "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + "physicalProcessorCount": 1, + "physicalCoreCount": 4, + "logicalCoreCount": 8, + "nominalFrequencyHz": 3100000000, + "maxFrequencyHz": 3100000000 + } + }, + "nested": [ + { + "job": { + "environment": { + "runtime": "net481" + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 10, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 11, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 12, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 20, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 21, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 22, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net70" + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 30, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 31, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 32, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 40, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 41, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 42, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default03.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default03.verified.txt new file mode 100644 index 0000000000..a940585663 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default03.verified.txt @@ -0,0 +1,94 @@ +BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + +Type = Bench, Runtime = Net70, Jit = RyuJit + +| Method | Center | Spread | +|:-------|--------:|--------:| +| Foo | 31.0 ns | 0.81 ns | +| Bar | 41.0 ns | 0.81 ns | + +{ + "engine": { + "name": "BenchmarkDotNet", + "version": "0.1729.0-mock" + }, + "host": { + "runtimeVersion": "Clr 4.0.x.mock", + "hasAttachedDebugger": false, + "hasRyuJit": true, + "configuration": "CONFIGURATION", + "dotNetSdkVersion": "1.0.x.mock", + "chronometerFrequency": 2531248, + "hardwareTimerKind": "Tsc", + "os": { + "display": "Microsoft Windows NT 10.0.x.mock" + }, + "cpu": { + "processorName": "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + "physicalProcessorCount": 1, + "physicalCoreCount": 4, + "logicalCoreCount": 8, + "nominalFrequencyHz": 3100000000, + "maxFrequencyHz": 3100000000 + } + }, + "nested": [ + { + "job": { + "environment": { + "runtime": "net70", + "jit": "ryuJit" + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 30, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 31, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 32, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 40, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 41, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 42, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default04.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default04.verified.txt new file mode 100644 index 0000000000..b9b8e6556f --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default04.verified.txt @@ -0,0 +1,155 @@ +BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + +Type = Bench + +@Net481LegacyJit: Runtime = Net481, Jit = LegacyJit +@Net70RyuJit: Runtime = Net70, Jit = RyuJit + +| Method | Job | Center | Spread | +|:-------|:----------------|--------:|--------:| +| Foo | Net481LegacyJit | 11.0 ns | 0.81 ns | +| Bar | Net481LegacyJit | 21.0 ns | 0.81 ns | +| Foo | Net70RyuJit | 31.0 ns | 0.81 ns | +| Bar | Net70RyuJit | 41.0 ns | 0.81 ns | + +{ + "engine": { + "name": "BenchmarkDotNet", + "version": "0.1729.0-mock" + }, + "host": { + "runtimeVersion": "Clr 4.0.x.mock", + "hasAttachedDebugger": false, + "hasRyuJit": true, + "configuration": "CONFIGURATION", + "dotNetSdkVersion": "1.0.x.mock", + "chronometerFrequency": 2531248, + "hardwareTimerKind": "Tsc", + "os": { + "display": "Microsoft Windows NT 10.0.x.mock" + }, + "cpu": { + "processorName": "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + "physicalProcessorCount": 1, + "physicalCoreCount": 4, + "logicalCoreCount": 8, + "nominalFrequencyHz": 3100000000, + "maxFrequencyHz": 3100000000 + } + }, + "nested": [ + { + "job": { + "environment": { + "runtime": "net481", + "jit": "legacyJit" + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 10, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 11, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 12, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 20, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 21, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 22, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net70", + "jit": "ryuJit" + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 30, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 31, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 32, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 40, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 41, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 42, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default05.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default05.verified.txt new file mode 100644 index 0000000000..0b54da76ec --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default05.verified.txt @@ -0,0 +1,1237 @@ +BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + +Type = Bench, Jit = RyuJit + +@HostProcessAffinity0: Runtime = HostProcess, Affinity = 0 +@Squid: Runtime = NotRecognized, Affinity = 1 +@MonoAffinity2: Runtime = Mono, Affinity = 2 +@Net461Affinity3: Runtime = Net461, Affinity = 3 +@Net462Affinity4: Runtime = Net462, Affinity = 4 +@Net47Affinity5: Runtime = Net47, Affinity = 5 +@Net471Affinity6: Runtime = Net471, Affinity = 6 +@Net472Affinity7: Runtime = Net472, Affinity = 7 +@Net48Affinity8: Runtime = Net48, Affinity = 8 +@Net481Affinity9: Runtime = Net481, Affinity = 9 +@Liger: Runtime = NetCoreApp20, Affinity = 10 +@Rat: Runtime = NetCoreApp21, Affinity = 11 +@Skate: Runtime = NetCoreApp22, Affinity = 12 +@Perch: Runtime = NetCoreApp30, Affinity = 13 +@Roach: Runtime = NetCoreApp31, Affinity = 14 +@Roe: Runtime = NetCoreApp50, Affinity = 15 +@Net50Affinity16: Runtime = Net50, Affinity = 16 +@Net60Affinity17: Runtime = Net60, Affinity = 17 +@Net70Affinity18: Runtime = Net70, Affinity = 18 +@Net80Affinity19: Runtime = Net80, Affinity = 19 + +| Method | Job | Center | Spread | +|:-------|:---------------------|----------:|--------:| +| Foo | HostProcessAffinity0 | 2.00 ns | 0.81 ns | +| Bar | HostProcessAffinity0 | 7.00 ns | 0.81 ns | +| Foo | Squid | 12.00 ns | 0.81 ns | +| Bar | Squid | 17.00 ns | 0.81 ns | +| Foo | MonoAffinity2 | 22.00 ns | 0.81 ns | +| Bar | MonoAffinity2 | 27.00 ns | 0.81 ns | +| Foo | Net461Affinity3 | 32.00 ns | 0.81 ns | +| Bar | Net461Affinity3 | 37.00 ns | 0.81 ns | +| Foo | Net462Affinity4 | 42.00 ns | 0.81 ns | +| Bar | Net462Affinity4 | 47.00 ns | 0.81 ns | +| Foo | Net47Affinity5 | 52.00 ns | 0.81 ns | +| Bar | Net47Affinity5 | 57.00 ns | 0.81 ns | +| Foo | Net471Affinity6 | 62.00 ns | 0.81 ns | +| Bar | Net471Affinity6 | 67.00 ns | 0.81 ns | +| Foo | Net472Affinity7 | 72.00 ns | 0.81 ns | +| Bar | Net472Affinity7 | 77.00 ns | 0.81 ns | +| Foo | Net48Affinity8 | 82.00 ns | 0.81 ns | +| Bar | Net48Affinity8 | 87.00 ns | 0.81 ns | +| Foo | Net481Affinity9 | 92.00 ns | 0.81 ns | +| Bar | Net481Affinity9 | 97.00 ns | 0.81 ns | +| Foo | Liger | 102.00 ns | 0.81 ns | +| Bar | Liger | 107.00 ns | 0.81 ns | +| Foo | Rat | 112.00 ns | 0.81 ns | +| Bar | Rat | 117.00 ns | 0.81 ns | +| Foo | Skate | 122.00 ns | 0.81 ns | +| Bar | Skate | 127.00 ns | 0.81 ns | +| Foo | Perch | 132.00 ns | 0.81 ns | +| Bar | Perch | 137.00 ns | 0.81 ns | +| Foo | Roach | 142.00 ns | 0.81 ns | +| Bar | Roach | 147.00 ns | 0.81 ns | +| Foo | Roe | 152.00 ns | 0.81 ns | +| Bar | Roe | 157.00 ns | 0.81 ns | +| Foo | Net50Affinity16 | 162.00 ns | 0.81 ns | +| Bar | Net50Affinity16 | 167.00 ns | 0.81 ns | +| Foo | Net60Affinity17 | 172.00 ns | 0.81 ns | +| Bar | Net60Affinity17 | 177.00 ns | 0.81 ns | +| Foo | Net70Affinity18 | 182.00 ns | 0.81 ns | +| Bar | Net70Affinity18 | 187.00 ns | 0.81 ns | +| Foo | Net80Affinity19 | 192.00 ns | 0.81 ns | +| Bar | Net80Affinity19 | 197.00 ns | 0.81 ns | + +{ + "engine": { + "name": "BenchmarkDotNet", + "version": "0.1729.0-mock" + }, + "host": { + "runtimeVersion": "Clr 4.0.x.mock", + "hasAttachedDebugger": false, + "hasRyuJit": true, + "configuration": "CONFIGURATION", + "dotNetSdkVersion": "1.0.x.mock", + "chronometerFrequency": 2531248, + "hardwareTimerKind": "Tsc", + "os": { + "display": "Microsoft Windows NT 10.0.x.mock" + }, + "cpu": { + "processorName": "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + "physicalProcessorCount": 1, + "physicalCoreCount": 4, + "logicalCoreCount": 8, + "nominalFrequencyHz": 3100000000, + "maxFrequencyHz": 3100000000 + } + }, + "nested": [ + { + "job": { + "environment": { + "runtime": "hostProcess", + "jit": "ryuJit", + "affinity": 0 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 1, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 2, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 3, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 6, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 7, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 8, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "notRecognized", + "jit": "ryuJit", + "affinity": 1 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 11, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 12, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 13, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 16, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 17, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 18, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "mono", + "jit": "ryuJit", + "affinity": 2 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 21, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 22, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 23, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 26, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 27, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 28, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net461", + "jit": "ryuJit", + "affinity": 3 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 31, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 32, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 33, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 36, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 37, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 38, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net462", + "jit": "ryuJit", + "affinity": 4 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 41, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 42, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 43, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 46, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 47, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 48, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net47", + "jit": "ryuJit", + "affinity": 5 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 51, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 52, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 53, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 56, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 57, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 58, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net471", + "jit": "ryuJit", + "affinity": 6 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 61, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 62, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 63, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 66, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 67, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 68, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net472", + "jit": "ryuJit", + "affinity": 7 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 71, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 72, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 73, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 76, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 77, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 78, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net48", + "jit": "ryuJit", + "affinity": 8 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 81, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 82, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 83, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 86, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 87, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 88, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net481", + "jit": "ryuJit", + "affinity": 9 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 91, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 92, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 93, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 96, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 97, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 98, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp20", + "jit": "ryuJit", + "affinity": 10 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 101, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 102, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 103, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 106, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 107, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 108, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp21", + "jit": "ryuJit", + "affinity": 11 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 111, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 112, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 113, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 116, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 117, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 118, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp22", + "jit": "ryuJit", + "affinity": 12 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 121, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 122, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 123, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 126, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 127, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 128, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp30", + "jit": "ryuJit", + "affinity": 13 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 131, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 132, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 133, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 136, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 137, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 138, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp31", + "jit": "ryuJit", + "affinity": 14 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 141, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 142, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 143, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 146, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 147, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 148, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp50", + "jit": "ryuJit", + "affinity": 15 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 151, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 152, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 153, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 156, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 157, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 158, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net50", + "jit": "ryuJit", + "affinity": 16 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 161, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 162, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 163, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 166, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 167, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 168, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net60", + "jit": "ryuJit", + "affinity": 17 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 171, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 172, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 173, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 176, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 177, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 178, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net70", + "jit": "ryuJit", + "affinity": 18 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 181, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 182, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 183, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 186, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 187, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 188, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net80", + "jit": "ryuJit", + "affinity": 19 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 191, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 192, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 193, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 196, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 197, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 198, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=params01.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=params01.verified.txt new file mode 100644 index 0000000000..f5f36bb3b7 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=params01.verified.txt @@ -0,0 +1,51 @@ +| Method | A | B | Center | +|:-------|---:|---:|--------:| +| Foo | 1 | 2 | 10.0 ms | +| Bar | 10 | 20 | 20.0 ms | + +{ + "nested": [ + { + "parameters": { + "a": 1, + "b": 2 + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 10, + "unit": "ms", + "iterationIndex": 0 + } + ] + } + ] + }, + { + "parameters": { + "a": 10, + "b": 20 + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 20, + "unit": "ms", + "iterationIndex": 0 + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=sort01.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=sort01.verified.txt new file mode 100644 index 0000000000..8c2956eb1a --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=sort01.verified.txt @@ -0,0 +1,35 @@ +| Method | Center | +|:-------|--------:| +| Bar | 20.0 ms | +| Foo | 10.0 ms | + +{ + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 10, + "unit": "ms", + "iterationIndex": 0 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 20, + "unit": "ms", + "iterationIndex": 0 + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/CpuInfoFormatterTests.cs b/tests/BenchmarkDotNet.Tests/Portability/Cpu/CpuInfoFormatterTests.cs deleted file mode 100644 index 83d14685a4..0000000000 --- a/tests/BenchmarkDotNet.Tests/Portability/Cpu/CpuInfoFormatterTests.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System.Text; -using System.Threading.Tasks; -using BenchmarkDotNet.Portability.Cpu; -using BenchmarkDotNet.Tests.Builders; -using VerifyXunit; -using Xunit; - -namespace BenchmarkDotNet.Tests.Portability.Cpu -{ - [Collection("VerifyTests")] - [UsesVerify] - public class CpuInfoFormatterTests - { - [Fact] - public Task FormatTest() - { - var captions = new StringBuilder(); - foreach (var processorName in new[] { null, "", "Intel" }) - foreach (var physicalProcessorCount in new int?[] { null, 0, 1, 2 }) - foreach (var physicalCoreCount in new int?[] { null, 0, 1, 2 }) - foreach (var logicalCoreCount in new int?[] { null, 0, 1, 2 }) - { - var mockCpuInfo = new CpuInfo(processorName, physicalProcessorCount, physicalCoreCount, logicalCoreCount, null, null, null); - captions.AppendLine(CpuInfoFormatter.Format(mockCpuInfo)); - } - - var settings = VerifySettingsFactory.Create(); - return Verifier.Verify(captions.ToString(), settings); - } - } -} diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/ProcCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Portability/Cpu/ProcCpuInfoParserTests.cs deleted file mode 100644 index 6d2faa423e..0000000000 --- a/tests/BenchmarkDotNet.Tests/Portability/Cpu/ProcCpuInfoParserTests.cs +++ /dev/null @@ -1,90 +0,0 @@ -using BenchmarkDotNet.Portability.Cpu; -using Perfolizer.Horology; -using Xunit; - -namespace BenchmarkDotNet.Tests.Portability.Cpu -{ - public class ProcCpuInfoParserTests - { - [Fact] - public void EmptyTest() - { - var parser = ProcCpuInfoParser.ParseOutput(string.Empty); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - - Assert.Null(parser.MaxFrequency); - Assert.Null(parser.MinFrequency); - } - - [Fact] - public void MalformedTest() - { - var parser = ProcCpuInfoParser.ParseOutput("malformedkey: malformedvalue\n\nmalformedkey2: malformedvalue2"); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - Assert.Null(parser.MaxFrequency); - Assert.Null(parser.MinFrequency); - } - - [Fact] - public void TwoProcessorWithDifferentCoresCountTest() - { - string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoProcessorWithDifferentCoresCount.txt"); - var parser = ProcCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Unknown processor with 2 cores and hyper threading, Unknown processor with 4 cores", parser.ProcessorName); - Assert.Equal(2, parser.PhysicalProcessorCount); - Assert.Equal(6, parser.PhysicalCoreCount); - Assert.Equal(8, parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - Assert.Equal(0.8 * Frequency.GHz, parser.MinFrequency); - Assert.Equal(2.5 * Frequency.GHz, parser.MaxFrequency); - } - - - [Fact] - public void RealOneProcessorTwoCoresTest() - { - string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoRealOneProcessorTwoCores.txt"); - var parser = ProcCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Core(TM) i5-6200U CPU @ 2.30GHz", parser.ProcessorName); - Assert.Equal(1, parser.PhysicalProcessorCount); - Assert.Equal(2, parser.PhysicalCoreCount); - Assert.Equal(4, parser.LogicalCoreCount); - Assert.Equal(2.3 * Frequency.GHz, parser.NominalFrequency); - Assert.Equal(0.8 * Frequency.GHz, parser.MinFrequency); - Assert.Equal(2.3 * Frequency.GHz, parser.MaxFrequency); - } - - [Fact] - public void RealOneProcessorFourCoresTest() - { - string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoRealOneProcessorFourCores.txt"); - var parser = ProcCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", parser.ProcessorName); - Assert.Equal(1, parser.PhysicalProcessorCount); - Assert.Equal(4, parser.PhysicalCoreCount); - Assert.Equal(8, parser.LogicalCoreCount); - Assert.Equal(2.5 * Frequency.GHz, parser.NominalFrequency); - Assert.Equal(0.8 * Frequency.GHz, parser.MinFrequency); - Assert.Equal(2.5 * Frequency.GHz, parser.MaxFrequency); - } - - [Theory] - [InlineData("Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", 2.50)] - [InlineData("Intel(R) Core(TM) i5-6200U CPU @ 2.30GHz", 2.30)] - [InlineData("Unknown processor with 2 cores and hyper threading, Unknown processor with 4 cores", 0)] - [InlineData("Intel(R) Core(TM) i5-2500 CPU @ 3.30GHz", 3.30)] - public void ParseFrequencyFromBrandStringTests(string brandString, double expectedGHz) - { - var frequency = ProcCpuInfoParser.ParseFrequencyFromBrandString(brandString); - Assert.Equal(Frequency.FromGHz(expectedGHz), frequency); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/SysctlCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Portability/Cpu/SysctlCpuInfoParserTests.cs deleted file mode 100644 index 5d2ba82520..0000000000 --- a/tests/BenchmarkDotNet.Tests/Portability/Cpu/SysctlCpuInfoParserTests.cs +++ /dev/null @@ -1,43 +0,0 @@ -using BenchmarkDotNet.Portability.Cpu; -using Perfolizer.Horology; -using Xunit; - -namespace BenchmarkDotNet.Tests.Portability.Cpu -{ - public class SysctlCpuInfoParserTests - { - [Fact] - public void EmptyTest() - { - var parser = SysctlCpuInfoParser.ParseOutput(string.Empty); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - } - - [Fact] - public void MalformedTest() - { - var parser = SysctlCpuInfoParser.ParseOutput("malformedkey=malformedvalue\n\nmalformedkey2=malformedvalue2"); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - } - - [Fact] - public void RealOneProcessorFourCoresTest() - { - string cpuInfo = TestHelper.ReadTestFile("SysctlRealOneProcessorFourCores.txt"); - var parser = SysctlCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz", parser.ProcessorName); - Assert.Equal(1, parser.PhysicalProcessorCount); - Assert.Equal(4, parser.PhysicalCoreCount); - Assert.Equal(8, parser.LogicalCoreCount); - Assert.Equal(2200 * Frequency.MHz, parser.NominalFrequency); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestHelper.cs b/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestHelper.cs deleted file mode 100644 index e4af5025c0..0000000000 --- a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestHelper.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System.IO; -using System.Reflection; - -namespace BenchmarkDotNet.Tests.Portability.Cpu -{ - public static class TestHelper - { - public static string ReadTestFile(string name) - { - var assembly = typeof(TestHelper).GetTypeInfo().Assembly; - string resourceName = $"{typeof(TestHelper).Namespace}.TestFiles.{name}"; - - using (var stream = assembly.GetManifestResourceStream(resourceName)) - using (var reader = new StreamReader(stream)) - { - return reader.ReadToEnd(); - } - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/WmicCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Portability/Cpu/WmicCpuInfoParserTests.cs deleted file mode 100644 index b1faa4b521..0000000000 --- a/tests/BenchmarkDotNet.Tests/Portability/Cpu/WmicCpuInfoParserTests.cs +++ /dev/null @@ -1,103 +0,0 @@ -using BenchmarkDotNet.Portability.Cpu; -using Perfolizer.Horology; -using Xunit; - -namespace BenchmarkDotNet.Tests.Portability.Cpu -{ - public class WmicCpuInfoParserTests - { - [Fact] - public void EmptyTest() - { - var parser = WmicCpuInfoParser.ParseOutput(string.Empty); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - } - - [Fact] - public void MalformedTest() - { - var parser = WmicCpuInfoParser.ParseOutput("malformedkey=malformedvalue\n\nmalformedkey2=malformedvalue2"); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - } - - [Fact] - public void RealTwoProcessorEightCoresTest() - { - const string cpuInfo = @" - -MaxClockSpeed=2400 -Name=Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz -NumberOfCores=8 -NumberOfLogicalProcessors=16 - - -MaxClockSpeed=2400 -Name=Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz -NumberOfCores=8 -NumberOfLogicalProcessors=16 - -"; - var parser = WmicCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz", parser.ProcessorName); - Assert.Equal(2, parser.PhysicalProcessorCount); - Assert.Equal(16, parser.PhysicalCoreCount); - Assert.Equal(32, parser.LogicalCoreCount); - Assert.Equal(2400 * Frequency.MHz, parser.MaxFrequency); - } - - [Fact] - public void RealTwoProcessorEightCoresWithWmicBugTest() - { - const string cpuInfo = - "\r\r\n" + - "\r\r\n" + - "MaxClockSpeed=3111\r\r\n" + - "Name=Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz\r\r\n" + - "NumberOfCores=8\r\r\n" + - "NumberOfLogicalProcessors=16\r\r\n" + - "\r\r\n" + - "\r\r\n" + - "MaxClockSpeed=3111\r\r\n" + - "Name=Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz\r\r\n" + - "NumberOfCores=8\r\r\n" + - "NumberOfLogicalProcessors=16\r\r\n" + - "\r\r\n" + - "\r\r\n" + - "\r\r\n"; - var parser = WmicCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz", parser.ProcessorName); - Assert.Equal(2, parser.PhysicalProcessorCount); - Assert.Equal(16, parser.PhysicalCoreCount); - Assert.Equal(32, parser.LogicalCoreCount); - Assert.Equal(3111 * Frequency.MHz, parser.MaxFrequency); - } - - [Fact] - public void RealOneProcessorFourCoresTest() - { - const string cpuInfo = @" - -MaxClockSpeed=2500 -Name=Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz -NumberOfCores=4 -NumberOfLogicalProcessors=8 - -"; - - var parser = WmicCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", parser.ProcessorName); - Assert.Equal(1, parser.PhysicalProcessorCount); - Assert.Equal(4, parser.PhysicalCoreCount); - Assert.Equal(8, parser.LogicalCoreCount); - Assert.Equal(2500 * Frequency.MHz, parser.MaxFrequency); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/HyperVTests.cs b/tests/BenchmarkDotNet.Tests/Portability/HyperVTests.cs index 8abe71fc64..8e2588bbfb 100644 --- a/tests/BenchmarkDotNet.Tests/Portability/HyperVTests.cs +++ b/tests/BenchmarkDotNet.Tests/Portability/HyperVTests.cs @@ -21,7 +21,7 @@ public void ContainsCorrectName() [InlineData("Dell", "ubuntu", false)] [InlineData("microsoft corporation", null, false)] [InlineData(null, "virtual machine", false)] - public void DetectsVirtualMachine(string manufacturer, string model, bool expectedResult) + public void DetectsVirtualMachine(string? manufacturer, string? model, bool expectedResult) { bool result = hypervisor.IsVirtualMachine(manufacturer, model); Assert.Equal(expectedResult, result); diff --git a/tests/BenchmarkDotNet.Tests/Portability/VMWareTests.cs b/tests/BenchmarkDotNet.Tests/Portability/VMWareTests.cs index 96758a97c2..f2e383ba85 100644 --- a/tests/BenchmarkDotNet.Tests/Portability/VMWareTests.cs +++ b/tests/BenchmarkDotNet.Tests/Portability/VMWareTests.cs @@ -20,7 +20,7 @@ public void ContainsCorrectName() [InlineData(null, "vmWare", true)] [InlineData("VMWare Inc", "redundant", false)] [InlineData("VMWare Inc", null, false)] - public void DetectsVirtualMachine(string manufacturer, string model, bool expectedResult) + public void DetectsVirtualMachine(string? manufacturer, string? model, bool expectedResult) { bool result = hypervisor.IsVirtualMachine(manufacturer, model); Assert.Equal(expectedResult, result); diff --git a/tests/BenchmarkDotNet.Tests/Portability/VirtualBoxTests.cs b/tests/BenchmarkDotNet.Tests/Portability/VirtualBoxTests.cs index 83e13ef500..56cf697fb0 100644 --- a/tests/BenchmarkDotNet.Tests/Portability/VirtualBoxTests.cs +++ b/tests/BenchmarkDotNet.Tests/Portability/VirtualBoxTests.cs @@ -18,7 +18,7 @@ public void ContainsCorrectName() [InlineData("redundant", "VirtualBox", true)] [InlineData("redundant", "vmware", false)] [InlineData("redundant", null, false)] - public void DetectsVirtualMachine(string manufacturer, string model, bool expectedResult) + public void DetectsVirtualMachine(string manufacturer, string? model, bool expectedResult) { bool result = hypervisor.IsVirtualMachine(manufacturer, model); Assert.Equal(expectedResult, result); diff --git a/tests/BenchmarkDotNet.Tests/ReflectionTests.cs b/tests/BenchmarkDotNet.Tests/ReflectionTests.cs index a32993a034..0d1ea69673 100644 --- a/tests/BenchmarkDotNet.Tests/ReflectionTests.cs +++ b/tests/BenchmarkDotNet.Tests/ReflectionTests.cs @@ -38,7 +38,7 @@ public void GetCorrectCSharpTypeNameSupportsGenericTypesPassedByReference() public class GenericByRef { - public void TheMethod(ref ValueTuple _) { } + public void TheMethod(ref (int, short) _) { } } [Fact] diff --git a/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs b/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs index 96c9519a14..f36066fa8a 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs @@ -48,7 +48,7 @@ public TestData(RatioStyle ratioStyle, int[] meanValues, int noise, string[] exp private static readonly IDictionary Data = new Dictionary { { "Percentage", new TestData(RatioStyle.Percentage, new[] { 100, 15, 115 }, 1, new[] { "baseline", "-85%", "+15%" }) }, - { "Trend", new TestData(RatioStyle.Trend, new[] { 100, 15, 115 }, 1, new[] { "baseline", "6.83x faster", "1.15x slower" }) } + { "Trend", new TestData(RatioStyle.Trend, new[] { 100, 15, 115 }, 1, new[] { "baseline", "6.84x faster", "1.15x slower" }) } }; [UsedImplicitly] diff --git a/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs b/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs index ebea88b5ad..fb15149aa3 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs @@ -1,6 +1,8 @@ using System.Linq; +using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Order; @@ -171,5 +173,311 @@ public void MissingValueInMetricColumnIsNA() // assert Assert.Equal(new[] { "-", "NA" }, actual); } + + #region Issue #2673 + [Fact] + public void DefaultExceptionDiagnoserConfig_WhenExceptionsIsNotZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var exceptionConfig = new ExceptionDiagnoserConfig(); + + var exceptionsFrequencyMetricDescriptor = new ExceptionDiagnoser.ExceptionsFrequencyMetricDescriptor(exceptionConfig); + + var metric = new Metric(exceptionsFrequencyMetricDescriptor, 5); + var metrics = new[] { metric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + var actual = table.Columns.First(c => c.Header == "Exceptions").Content; + + // assert + Assert.Equal(new[] { "5.0000", "5.0000" }, actual); + } + + [Fact] + public void DefaultExceptionDiagnoserConfig_WhenExceptionsIsZero() + { + + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var exceptionConfig = new ExceptionDiagnoserConfig(); + + var exceptionsFrequencyMetricDescriptor = new ExceptionDiagnoser.ExceptionsFrequencyMetricDescriptor(exceptionConfig); + + var metric = new Metric(exceptionsFrequencyMetricDescriptor, 0); + var metrics = new[] { metric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + var actual = table.Columns.First(c => c.Header == "Exceptions").Content; + + // assert + Assert.Equal(new[] { "-", "-" }, actual); + } + + [Fact] + public void HideExceptionDiagnoserConfig_WhenExceptionsIsNotZero() + { + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var exceptionConfig = new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: false); + + var exceptionsFrequencyMetricDescriptor = new ExceptionDiagnoser.ExceptionsFrequencyMetricDescriptor(exceptionConfig); + + var metric = new Metric(exceptionsFrequencyMetricDescriptor, 5); + var metrics = new[] { metric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + var actual = table.Columns.First(c => c.Header == "Exceptions").Content; + + // assert + Assert.Equal(new[] { "5.0000", "5.0000" }, actual); + } + + [Fact] + public void HideExceptionDiagnoserConfig_WhenExceptionsIsZero() + { + + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var exceptionConfig = new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: false); + + var exceptionsFrequencyMetricDescriptor = new ExceptionDiagnoser.ExceptionsFrequencyMetricDescriptor(exceptionConfig); + + var metric = new Metric(exceptionsFrequencyMetricDescriptor, 0); + var metrics = new[] { metric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + var isExist = table.Columns.Any(c => c.Header == "Exceptions"); + + // assert + Assert.False(isExist); + } + + [Fact] + public void DefaultThreadingDiagnoserConfig_WhenDescriptorValuesAreNotZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 5); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 5); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + var lockContentionCount = table.Columns.FirstOrDefault(c => c.Header == "Lock Contentions").Content; + var completedWorkItemCount = table.Columns.FirstOrDefault(c => c.Header == "Completed Work Items").Content; + + // assert + Assert.Equal(new[] { "5.0000", "5.0000" }, lockContentionCount); + Assert.Equal(new[] { "5.0000", "5.0000" }, completedWorkItemCount); + } + + [Fact] + public void DefaultThreadingDiagnoserConfig_WhenDescriptorValuesAreZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 0); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 0); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + var lockContentionCount = table.Columns.FirstOrDefault(c => c.Header == "Lock Contentions").Content; + var completedWorkItemCount = table.Columns.FirstOrDefault(c => c.Header == "Completed Work Items").Content; + + // assert + Assert.Equal(new[] { "-", "-" }, lockContentionCount); + Assert.Equal(new[] { "-", "-" }, completedWorkItemCount); + } + + [Fact] + public void HideLockContentionCountThreadingDiagnoserConfig_WhenDescriptorValuesAreZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayLockContentionWhenZero: false); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 0); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 0); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Null(lockContentionCount); + Assert.Equal(new[] { "-", "-" }, completedWorkItemCount); + } + + [Fact] + public void HideLockContentionCountThreadingDiagnoserConfig_WhenDescriptorValuesAreNotZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayLockContentionWhenZero: false); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 5); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 5); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Equal(new[] { "5.0000", "5.0000" }, lockContentionCount); + Assert.Equal(new[] { "5.0000", "5.0000" }, completedWorkItemCount); + } + + [Fact] + public void HideCompletedWorkItemCountThreadingDiagnoserConfig_WhenDescriptorValuesAreZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 0); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 0); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Null(completedWorkItemCount); + Assert.Equal(new[] { "-", "-" }, lockContentionCount); + } + + [Fact] + public void HideCompletedWorkItemCountThreadingDiagnoserConfig_WhenDescriptorValuesAreNotZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 5); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 5); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Equal(new[] { "5.0000", "5.0000" }, lockContentionCount); + Assert.Equal(new[] { "5.0000", "5.0000" }, completedWorkItemCount); + } + + [Fact] + public void HideThreadingDiagnoserConfigs_WhenDescriptorValuesAreZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false, displayLockContentionWhenZero: false); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 0); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 0); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Null(lockContentionCount); + Assert.Null(completedWorkItemCount); + } + + [Fact] + public void DisplayThreadingDiagnoserConfigs_WhenDescriptorValuesAreZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: true, displayLockContentionWhenZero: true); + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 0); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 0); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Equal(new[] { "-", "-" }, lockContentionCount); + Assert.Equal(new[] { "-", "-" }, completedWorkItemCount); + } + #endregion } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs b/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs index 8952accb40..92811e5933 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs @@ -35,7 +35,7 @@ public void SummaryWithFailureReportDoesNotThrowNre() private static IConfig CreateConfig() { // We use runtime as selector later. It is chosen as selector just to be close to initial issue. Nothing particularly special about it. - Job coreJob = new Job(Job.Default).WithRuntime(CoreRuntime.Core20).ApplyAndFreeze(RunMode.Dry); + Job coreJob = new Job(Job.Default).WithRuntime(CoreRuntime.Core80).ApplyAndFreeze(RunMode.Dry); Job clrJob = new Job(Job.Default).WithRuntime(ClrRuntime.Net462).ApplyAndFreeze(RunMode.Dry); return ManualConfig.Create(DefaultConfig.Instance).AddJob(coreJob).AddJob(clrJob); } diff --git a/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs b/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs index 4eb72b5d59..4b3e92d95d 100644 --- a/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs @@ -135,7 +135,7 @@ public void JobMutatorsApplySettingsToAllNonMutatorJobs() typeof(WithMutator), DefaultConfig.Instance .AddJob(Job.Default.WithRuntime(ClrRuntime.Net462)) - .AddJob(Job.Default.WithRuntime(CoreRuntime.Core21))); + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80))); Assert.Equal(2, info.BenchmarksCases.Length); Assert.All(info.BenchmarksCases, benchmark => Assert.Equal(int.MaxValue, benchmark.Job.Run.MaxIterationCount)); diff --git a/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs b/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs index e04409beb9..4ace61168b 100644 --- a/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs @@ -95,6 +95,7 @@ public void CustomClrBuildJobsAreGroupedByVersion() } [Fact] + [System.Obsolete] public void CustomNuGetJobsAreGroupedByPackageVersion() { var config = ManualConfig.Create(DefaultConfig.Instance) diff --git a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs new file mode 100644 index 0000000000..05ce92b2cf --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs @@ -0,0 +1,359 @@ +using System; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Running; +using Xunit; +using System.Reflection; +using System.Reflection.Emit; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using System.Runtime.InteropServices; + +namespace BenchmarkDotNet.Tests.Running +{ + public class RunningEmptyBenchmarkTests + { + #region BenchmarkRunner Methods Overview + /* + * Available BenchmarkRunner.Run methods: + * 1. Generic Type: + * - BenchmarkRunner.Run(IConfig? config = null, string[]? args = null) + * 2. Type-based: + * - BenchmarkRunner.Run(Type type, IConfig? config = null, string[]? args = null) + * - BenchmarkRunner.Run(Type[] types, IConfig? config = null, string[]? args = null) + * - BenchmarkRunner.Run(Type type, MethodInfo[] methods, IConfig? config = null) + * 3. Assembly-based: + * - BenchmarkRunner.Run(Assembly assembly, IConfig? config = null, string[]? args = null) + * 4. BenchmarkRunInfo-based: + * - BenchmarkRunner.Run(BenchmarkRunInfo benchmarkRunInfo) + * - BenchmarkRunner.Run(BenchmarkRunInfo[] benchmarkRunInfos) + * 5. Deprecated methods: + * - BenchmarkRunner.RunUrl(string url, IConfig? config = null) + * - BenchmarkRunner.RunSource(string source, IConfig? config = null) + */ + #endregion + #region Generic Type Tests + /// + /// Tests for BenchmarkRunner.Run method + /// + [Theory] + [InlineData(null)] + //[InlineData(new object[] { new string[] { " " } })] + public void GenericTypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(config, args); + if (args == null) + { + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + } + + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + + [Theory] + [InlineData(null)] + //[InlineData(new object[] { new string[] { " " } })] + public void GenericTypeWithBenchmarkAttribute_RunsSuccessfully(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(config, args); + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + #endregion + #region Type-based Tests + /// + /// Tests for BenchmarkRunner.Run(Type) method + /// + [Theory] + [InlineData(null)] + //[InlineData(new object[] { new string[] { " " } })] + public void TypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + + var summary = BenchmarkRunner.Run(typeof(EmptyBenchmark), config, args); + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void TypeWithBenchmarkAttribute_RunsSuccessfully(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(typeof(NotEmptyBenchmark), config, args); + Assert.False(summaries.HasCriticalValidationErrors); + Assert.DoesNotContain(summaries.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + + /// + /// Tests for BenchmarkRunner.Run(Type[]) method + /// + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void TypesWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { typeof(EmptyBenchmark), typeof(EmptyBenchmark2) }, config, args); + if (args != null) + { + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark2)), logger.GetLog()); + } + else + { + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark2))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark2)), logger.GetLog()); + } + + + } + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void TypesWithBenchmarkAttribute_RunsSuccessfully(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { typeof(NotEmptyBenchmark) }, config, args); + var summary = summaries[0]; + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + #endregion + #region BenchmarkRunInfo Tests + /// + /// Tests for BenchmarkRunner.Run(BenchmarkRunInfo) method + /// + [Fact] + public void BenchmarkRunInfoWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute() + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), config)); + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + + [Fact] + public void BenchmarkRunInfoWithBenchmarkAttribute_RunsSuccessfully() + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), config)); + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + + /// + /// Tests for BenchmarkRunner.Run(BenchmarkRunInfo[]) method + /// + [Fact] + public void BenchmarkRunInfosWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute() + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { + BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), config), + BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark2), config) + }); + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark2))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark2)), logger.GetLog()); + } + + [Fact] + public void BenchmarkRunInfosWithBenchmarkAttribute_RunsSuccessfully() + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), config) }); + var summary = summaries[0]; + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + #endregion + #region Mixed Types Tests + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void MixedTypes_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { typeof(EmptyBenchmark), typeof(NotEmptyBenchmark) }, config, args); + if (args != null) + { + Assert.Contains(GetExpandedValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + else + { + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + } + #endregion + #region Assembly Tests + // In this tests there is no config and logger because the logger is initiated at CreateCompositeLogger when the BenchmarkRunInfo[] is empty + // those cannot be inserted using config + [Theory] + + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void AssemblyWithoutBenchmarks_ThrowsValidationError_WhenNoBenchmarksFound(string[]? args) + { + + // Create a mock assembly with no benchmark types + var assemblyName = new AssemblyName("MockAssembly"); + var assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.Run); + var moduleBuilder = assemblyBuilder.DefineDynamicModule("MockModule"); + // Create a simple type in the assembly (no benchmarks) + var typeBuilder = moduleBuilder.DefineType("MockType", TypeAttributes.Public); + typeBuilder.CreateType(); + + Summary[] summaries = null; + if (args != null) + { + GetConfigWithLogger(out var logger, out var config); + summaries = BenchmarkRunner.Run(assemblyBuilder, config, args); + Assert.Contains(GetAssemblylValidationError(assemblyBuilder), logger.GetLog()); + } + else + { + summaries = BenchmarkRunner.Run(assemblyBuilder, null, args); + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetGeneralValidationError()); + } + } + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void AssemblyWithBenchmarks_RunsSuccessfully_WhenBenchmarkAttributePresent(string[]? args) + { + // Skip test on .NET Framework 4.6.2 + if (RuntimeInformation.FrameworkDescription.Contains(".NET Framework 4")) + return; + + // Create a mock assembly with benchmark types + var assemblyName = new AssemblyName("MockAssemblyWithBenchmarks"); + var assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.Run); + var moduleBuilder = assemblyBuilder.DefineDynamicModule("MockModule"); + + // Create a benchmark type + var benchmarkTypeBuilder = moduleBuilder.DefineType("MockBenchmark", TypeAttributes.Public); + var benchmarkMethod = benchmarkTypeBuilder.DefineMethod("Benchmark", MethodAttributes.Public, typeof(void), Type.EmptyTypes); + + // Generate method body + var ilGenerator = benchmarkMethod.GetILGenerator(); + ilGenerator.Emit(OpCodes.Ret); // Just return from the method + + var benchmarkAttributeCtor = typeof(BenchmarkAttribute).GetConstructor(new[] { typeof(int), typeof(string) }); + if (benchmarkAttributeCtor == null) + throw new InvalidOperationException("Could not find BenchmarkAttribute constructor"); + benchmarkMethod.SetCustomAttribute(new CustomAttributeBuilder( + benchmarkAttributeCtor, + new object[] { 0, "" })); + benchmarkTypeBuilder.CreateType(); + + Summary[] summaries = null; + if (args != null) + { + GetConfigWithLogger(out var logger, out var config); + summaries = BenchmarkRunner.Run(assemblyBuilder, config, args); + Assert.DoesNotContain(GetAssemblylValidationError(assemblyBuilder), logger.GetLog()); + } + else + { + summaries = BenchmarkRunner.Run(assemblyBuilder); + var summary = summaries[0]; + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetGeneralValidationError()); + } + } + #endregion + #region Helper Methods + private string GetValidationErrorForType(Type type) + { + return $"No [Benchmark] attribute found on '{type.Name}' benchmark case."; + } + + private string GetAssemblylValidationError(Assembly assembly) + { + return $"No [Benchmark] attribute found on '{assembly.GetName().Name}' assembly."; + } + + private string GetExpandedValidationErrorForType(Type type) + { + return $"Type {type} is invalid. Only public, non-generic (closed generic types with public parameterless ctors are supported), non-abstract, non-sealed, non-static types with public instance [Benchmark] method(s) are supported."; + } + + private string GetGeneralValidationError() + { + return $"No benchmarks were found."; + } + + private void GetConfigWithLogger(out AccumulationLogger logger, out ManualConfig manualConfig) + { + logger = new AccumulationLogger(); + manualConfig = ManualConfig.CreateEmpty() + .AddLogger(logger) + .AddColumnProvider(DefaultColumnProviders.Instance); + } + + #endregion + #region Test Classes + public class EmptyBenchmark + { + } + + public class EmptyBenchmark2 + { + } + + [SimpleJob(launchCount: 1, warmupCount: 1, iterationCount: 1, invocationCount: 1, id: "QuickJob")] + public class NotEmptyBenchmark + { + [Benchmark] + public void Benchmark() + { + var sum = 0; + for (int i = 0; i < 1; i++) + { + sum += i; + } + } + } + #endregion + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs b/tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs index e49bd344e1..f6286720fd 100644 --- a/tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs +++ b/tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.IO; +using BenchmarkDotNet.Detectors; using Xunit; namespace BenchmarkDotNet.Tests @@ -17,6 +18,10 @@ public class RuntimeVersionDetectionTests [InlineData(".NETCoreApp,Version=v3.0", RuntimeMoniker.NetCoreApp30, "netcoreapp3.0")] [InlineData(".NETCoreApp,Version=v3.1", RuntimeMoniker.NetCoreApp31, "netcoreapp3.1")] [InlineData(".NETCoreApp,Version=v5.0", RuntimeMoniker.Net50, "net5.0")] + [InlineData(".NETCoreApp,Version=v6.0", RuntimeMoniker.Net60, "net6.0")] + [InlineData(".NETCoreApp,Version=v7.0", RuntimeMoniker.Net70, "net7.0")] + [InlineData(".NETCoreApp,Version=v8.0", RuntimeMoniker.Net80, "net8.0")] + [InlineData(".NETCoreApp,Version=v9.0", RuntimeMoniker.Net90, "net9.0")] [InlineData(".NETCoreApp,Version=v123.0", RuntimeMoniker.NotRecognized, "net123.0")] public void TryGetVersionFromFrameworkNameHandlesValidInput(string frameworkName, RuntimeMoniker expectedTfm, string expectedMsBuildMoniker) { @@ -33,7 +38,7 @@ public void TryGetVersionFromFrameworkNameHandlesValidInput(string frameworkName [InlineData("")] [InlineData(".NETCoreApp,Version=v")] [InlineData("just wrong")] - public void TryGetVersionFromFrameworkNameHandlesInvalidInput(string frameworkName) + public void TryGetVersionFromFrameworkNameHandlesInvalidInput(string? frameworkName) { Assert.False(CoreRuntime.TryGetVersionFromFrameworkName(frameworkName, out _)); } @@ -60,7 +65,7 @@ public void TryGetVersionFromProductInfoHandlesValidInput(RuntimeMoniker expecte [InlineData("", "")] [InlineData("not", "ok")] [InlineData("Microsoft .NET Framework", "4.6.26614.01 @BuiltBy: dlab14-DDVSOWINAGE018 @Commit: a536e7eec55c538c94639cefe295aa672996bf9b")] // this is an actual output for 2.0 but it simply does not contain enough info - public void TryGetVersionFromProductInfoHandlesInvalidInput(string productName, string productVersion) + public void TryGetVersionFromProductInfoHandlesInvalidInput(string? productName, string? productVersion) { Assert.False(CoreRuntime.TryGetVersionFromProductInfo(productVersion, productName, out _)); } @@ -110,7 +115,7 @@ public void CurrentRuntimeIsProperlyRecognized() var runtime = RuntimeInformation.GetCurrentRuntime(); #if NETFRAMEWORK - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) Assert.True(runtime is ClrRuntime); else Assert.True(runtime is MonoRuntime); @@ -124,6 +129,8 @@ public void CurrentRuntimeIsProperlyRecognized() Assert.True(runtime is CoreRuntime coreRuntime && coreRuntime.RuntimeMoniker == RuntimeMoniker.NetCoreApp31); #elif NETCOREAPP5_0 Assert.True(runtime is CoreRuntime coreRuntime && coreRuntime.RuntimeMoniker == RuntimeMoniker.NetCoreApp50); +#elif NET8_0 + Assert.True(runtime is CoreRuntime coreRuntime && coreRuntime.RuntimeMoniker == RuntimeMoniker.Net80); #endif } } diff --git a/tests/BenchmarkDotNet.Tests/SimpleJsonTests.cs b/tests/BenchmarkDotNet.Tests/SimpleJsonTests.cs index 2bb57fe436..fd599aea5b 100644 --- a/tests/BenchmarkDotNet.Tests/SimpleJsonTests.cs +++ b/tests/BenchmarkDotNet.Tests/SimpleJsonTests.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; using System.Linq; using Xunit; -using JsonSerializer = SimpleJson.SimpleJson; +using JsonSerializer = SimpleJson.SimpleJsonSerializer; namespace BenchmarkDotNet.Tests { diff --git a/tests/BenchmarkDotNet.Tests/SizeUnitTests.cs b/tests/BenchmarkDotNet.Tests/SizeUnitTests.cs deleted file mode 100644 index e8cfbd5989..0000000000 --- a/tests/BenchmarkDotNet.Tests/SizeUnitTests.cs +++ /dev/null @@ -1,63 +0,0 @@ -using BenchmarkDotNet.Columns; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests -{ - public class SizeUnitTests - { - private readonly ITestOutputHelper output; - - public SizeUnitTests(ITestOutputHelper output) - { - this.output = output; - } - - - [Fact] - public void ConvertTest() - { - CheckConvertTwoWay(1024, SizeUnit.B, 1, SizeUnit.KB); - CheckConvertTwoWay(1024, SizeUnit.KB, 1, SizeUnit.MB); - CheckConvertTwoWay(1024, SizeUnit.MB, 1, SizeUnit.GB); - CheckConvertTwoWay(1024, SizeUnit.GB, 1, SizeUnit.TB); - CheckConvertTwoWay(1024L * 1024 * 1024 * 1024, SizeUnit.B, 1, SizeUnit.TB); - } - - [Theory] - [InlineData("0 B", 0)] - [InlineData("1 B", 1)] - [InlineData("10 B", 10)] - [InlineData("100 B", 100)] - [InlineData("1000 B", 1000)] - [InlineData("1023 B", 1023)] - [InlineData("1 KB", 1024)] - [InlineData("1 KB", 1025)] - [InlineData("1.07 KB", 1100)] - [InlineData("1.5 KB", 1024 + 512)] - [InlineData("10 KB", 10 * 1024)] - [InlineData("1023 KB", 1023 * 1024)] - [InlineData("1 MB", 1024 * 1024)] - [InlineData("1 GB", 1024 * 1024 * 1024)] - [InlineData("1 TB", 1024L * 1024 * 1024 * 1024)] - public void SizeUnitFormattingTest(string expected, long bytes) - { - Assert.Equal(expected, SizeValue.FromBytes(bytes).ToString(TestCultureInfo.Instance)); - } - - private void CheckConvertTwoWay(long value1, SizeUnit unit1, long value2, SizeUnit unit2) - { - CheckConvertOneWay(value1, unit1, value2, unit2); - CheckConvertOneWay(value2, unit2, value1, unit1); - } - - private void CheckConvertOneWay(long value1, SizeUnit unit1, long value2, SizeUnit unit2) - { - double convertedValue2 = SizeUnit.Convert(value1, unit1, unit2); - output.WriteLine($"Expected: {value1} {unit1.Name} = {value2} {unit2.Name}"); - output.WriteLine($"Actual: {value1} {unit1.Name} = {convertedValue2} {unit2.Name}"); - output.WriteLine(""); - Assert.Equal(value2, convertedValue2, 4); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/SourceCodeHelperTests.cs b/tests/BenchmarkDotNet.Tests/SourceCodeHelperTests.cs index 846d59faa9..6e8e504a60 100644 --- a/tests/BenchmarkDotNet.Tests/SourceCodeHelperTests.cs +++ b/tests/BenchmarkDotNet.Tests/SourceCodeHelperTests.cs @@ -24,7 +24,7 @@ public class SourceCodeHelperTests [InlineData(0.123f, "0.123f")] [InlineData(0.123d, "0.123d")] [InlineData(BindingFlags.Public, "(System.Reflection.BindingFlags)(16)")] - public void ToSourceCodeSimpleTest(object original, string expected) + public void ToSourceCodeSimpleTest(object? original, string expected) { string actual = SourceCodeHelper.ToSourceCode(original); output.WriteLine("ORIGINAL : " + original + " (" + original?.GetType() + ")"); diff --git a/tests/BenchmarkDotNet.Tests/StringExtensionsTests.cs b/tests/BenchmarkDotNet.Tests/StringExtensionsTests.cs index 2dbaf0f6bc..a03f308ad6 100644 --- a/tests/BenchmarkDotNet.Tests/StringExtensionsTests.cs +++ b/tests/BenchmarkDotNet.Tests/StringExtensionsTests.cs @@ -43,7 +43,7 @@ public void HtmlEncodeCharacters() [InlineData(" a ", " a")] [InlineData(" a ", " a")] [InlineData(" \r\n a \r\n", " a")] - public void AppendArgumentMakesSureOneSpaceBeforeStringArgument(string input, string expectedOutput) + public void AppendArgumentMakesSureOneSpaceBeforeStringArgument(string? input, string expectedOutput) { var stringBuilder = new StringBuilder(); var result = stringBuilder.AppendArgument(input).ToString(); @@ -58,7 +58,7 @@ public void AppendArgumentMakesSureOneSpaceBeforeStringArgument(string input, st [InlineData("http://test.com/ ", " http://test.com/")] [InlineData(" http://test.com/ ", " http://test.com/")] [InlineData("\r\n http://test.com/ \r\n", " http://test.com/")] - public void AppendArgumentMakesSureOneSpaceBeforeObjectArgument(string input, string expectedOutput) + public void AppendArgumentMakesSureOneSpaceBeforeObjectArgument(string? input, string expectedOutput) { Uri uri = input != null ? new Uri(input) : null; // Use Uri for our object type since that is what is used in code var stringBuilder = new StringBuilder(); diff --git a/tests/BenchmarkDotNet.Tests/SummaryStyleTests.cs b/tests/BenchmarkDotNet.Tests/SummaryStyleTests.cs index e627bf7db9..e75729d584 100644 --- a/tests/BenchmarkDotNet.Tests/SummaryStyleTests.cs +++ b/tests/BenchmarkDotNet.Tests/SummaryStyleTests.cs @@ -3,6 +3,7 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Reports; using Perfolizer.Horology; +using Perfolizer.Metrology; using Xunit; namespace BenchmarkDotNet.Tests diff --git a/tests/BenchmarkDotNet.Tests/TimeUnitTests.cs b/tests/BenchmarkDotNet.Tests/TimeUnitTests.cs deleted file mode 100644 index 6f7a26c9b0..0000000000 --- a/tests/BenchmarkDotNet.Tests/TimeUnitTests.cs +++ /dev/null @@ -1,63 +0,0 @@ -using Perfolizer.Horology; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests -{ - public class TimeUnitTests - { - private readonly ITestOutputHelper output; - - public TimeUnitTests(ITestOutputHelper output) - { - this.output = output; - } - - [Fact] - public void ConvertTest() - { - CheckConvertTwoWay(1000, TimeUnit.Nanosecond, 1, TimeUnit.Microsecond); - CheckConvertTwoWay(1000, TimeUnit.Microsecond, 1, TimeUnit.Millisecond); - CheckConvertTwoWay(1000, TimeUnit.Millisecond, 1, TimeUnit.Second); - CheckConvertTwoWay(60, TimeUnit.Second, 1, TimeUnit.Minute); - CheckConvertTwoWay(60, TimeUnit.Minute, 1, TimeUnit.Hour); - CheckConvertTwoWay(24, TimeUnit.Hour, 1, TimeUnit.Day); - } - - [Fact] - public void GetBestTimeUnitTest() - { - CheckGetBestTimeUnit(TimeUnit.Nanosecond); - CheckGetBestTimeUnit(TimeUnit.Nanosecond, 1.0); - CheckGetBestTimeUnit(TimeUnit.Nanosecond, 100.0); - CheckGetBestTimeUnit(TimeUnit.Microsecond, 1.0 * 1000); - CheckGetBestTimeUnit(TimeUnit.Microsecond, 100.0 * 1000); - CheckGetBestTimeUnit(TimeUnit.Millisecond, 1.0 * 1000 * 1000); - CheckGetBestTimeUnit(TimeUnit.Millisecond, 100.0 * 1000 * 1000); - CheckGetBestTimeUnit(TimeUnit.Second, 1.0 * 1000 * 1000 * 1000); - CheckGetBestTimeUnit(TimeUnit.Second, 100.0 * 1000 * 1000 * 1000); - CheckGetBestTimeUnit(TimeUnit.Day, double.MaxValue); - } - - private void CheckGetBestTimeUnit(TimeUnit timeUnit, params double[] values) - { - output.WriteLine($"Best TimeUnit for ({string.Join(";", values)})ns is {timeUnit.Description}"); - Assert.Equal(timeUnit.Name, TimeUnit.GetBestTimeUnit(values).Name); - } - - private void CheckConvertTwoWay(double value1, TimeUnit unit1, double value2, TimeUnit unit2) - { - CheckConvertOneWay(value1, unit1, value2, unit2); - CheckConvertOneWay(value2, unit2, value1, unit1); - } - - private void CheckConvertOneWay(double value1, TimeUnit unit1, double value2, TimeUnit unit2) - { - double convertedValue2 = TimeUnit.Convert(value1, unit1, unit2); - output.WriteLine($"Expected: {value1} {unit1.Name} = {value2} {unit2.Name}"); - output.WriteLine($"Actual: {value1} {unit1.Name} = {convertedValue2} {unit2.Name}"); - output.WriteLine(""); - Assert.Equal(value2, convertedValue2, 4); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Validators/CompilationValidatorTests.cs b/tests/BenchmarkDotNet.Tests/Validators/CompilationValidatorTests.cs index 18d947acea..683e258bbe 100644 --- a/tests/BenchmarkDotNet.Tests/Validators/CompilationValidatorTests.cs +++ b/tests/BenchmarkDotNet.Tests/Validators/CompilationValidatorTests.cs @@ -7,6 +7,7 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Validators; using Xunit; +using System.Reflection; namespace BenchmarkDotNet.Tests.Validators { @@ -22,9 +23,7 @@ public void BenchmarkedMethodNameMustNotContainWhitespaces() new[] { BenchmarkCase.Create( - new Descriptor( - typeof(CompilationValidatorTests), - method.Method), + new Descriptor(typeof(CompilationValidatorTests), method.Method), Job.Dry, null, config @@ -48,9 +47,7 @@ public void BenchmarkedMethodNameMustNotUseCsharpKeywords() new[] { BenchmarkCase.Create( - new Descriptor( - typeof(CompilationValidatorTests), - method.Method), + new Descriptor(typeof(CompilationValidatorTests), method.Method), Job.Dry, null, config) @@ -63,6 +60,29 @@ public void BenchmarkedMethodNameMustNotUseCsharpKeywords() "Benchmarked method `typeof` contains illegal character(s) or uses C# keyword. Please use `[]` to set custom display name.")); } + [Theory] + /* BenchmarkDotNet can only benchmark public unsealed classes*/ + [InlineData(typeof(BenchMarkPublicClass), false)] + [InlineData(typeof(BenchMarkPublicClass.PublicNestedClass), false)] + [InlineData(typeof(SealedClass.PublicNestedClass), false)] + [InlineData(typeof(OuterClass.PublicNestedClass), true)] + [InlineData(typeof(SealedClass), true)] + [InlineData(typeof(MyPrivateClass), true)] + [InlineData(typeof(MyPublicProtectedClass), true)] + [InlineData(typeof(MyPrivateProtectedClass), true)] + [InlineData(typeof(MyProtectedInternalClass), true)] + [InlineData(typeof(MyInternalClass), true)] + [InlineData(typeof(OuterClass), true)] + [InlineData(typeof(OuterClass.InternalNestedClass), true)] + [InlineData(typeof(BenchMarkPublicClass.InternalNestedClass), true)] + /* Generics Remaining */ + public void Benchmark_Class_Modifers_Must_Be_Public(Type type, bool hasErrors) + { + var validationErrors = CompilationValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(type)); + + Assert.Equal(hasErrors, validationErrors.Any()); + } + [Theory] [InlineData(typeof(BenchmarkClassWithStaticMethod), true)] [InlineData(typeof(BenchmarkClass), false)] @@ -119,7 +139,17 @@ protected internal class ProtectedInternalClass { protected internal class ProtectedInternalNestedClass { } } - } + + private class MyPrivateClass{ [Benchmark] public void PublicMethod(){} } + + protected class MyPublicProtectedClass{ [Benchmark] public void PublicMethod(){} } + + private protected class MyPrivateProtectedClass{ [Benchmark] public void PublicMethod(){} } + + internal class MyInternalClass{ [Benchmark] public void PublicMethod(){} } + + protected internal class MyProtectedInternalClass{ [Benchmark] public void PublicMethod() { } } + } public class BenchmarkClassWithStaticMethod { @@ -142,4 +172,29 @@ internal class InternalClass { internal class InternalNestedClass { } } + + public sealed class SealedClass + { + [Benchmark] public void PublicMethod() { } + + public class PublicNestedClass { [Benchmark] public void PublicMethod() { } } + } + + internal class OuterClass + { + [Benchmark] public void PublicMethod(){} + + internal class InternalNestedClass { [Benchmark] public void PublicMethod() { } } + + public class PublicNestedClass { [Benchmark] public void PublicMethod() { } } + } + + public class BenchMarkPublicClass + { + [Benchmark] public void PublicMethod(){} + + public class PublicNestedClass { [Benchmark] public void PublicMethod() { } } + + internal class InternalNestedClass { [Benchmark] public void PublicMethod() { } } + } } diff --git a/tests/BenchmarkDotNet.Tests/Validators/ConfigCompatibilityValidatorTests.cs b/tests/BenchmarkDotNet.Tests/Validators/ConfigCompatibilityValidatorTests.cs deleted file mode 100644 index fc73793980..0000000000 --- a/tests/BenchmarkDotNet.Tests/Validators/ConfigCompatibilityValidatorTests.cs +++ /dev/null @@ -1,180 +0,0 @@ -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Columns; -using BenchmarkDotNet.Order; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Tests.Loggers; -using BenchmarkDotNet.Validators; -using System.Linq; -using BenchmarkDotNet.Portability; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests.Validators -{ - public class ConfigCompatibilityValidatorTests - { - private ITestOutputHelper Output { get; } - - public ConfigCompatibilityValidatorTests(ITestOutputHelper output) => Output = output; - - [Fact] - public void RunningBenchmarksWithIncompatibleConfigsMustFailWithCriticalError() - { - if (!KnownIssue.Issue2299.IsFixed && RuntimeInformation.IsMono && RuntimeInformation.IsLinux()) - { - Output.WriteLine(KnownIssue.Issue2299.IgnoreMessage); - return; - } - - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); - var summary = - BenchmarkSwitcher - .FromTypes(new[] { typeof(BenchmarkClassWithExtraOrderer1), typeof(BenchmarkClassWithExtraOrderer2) }) - .RunAllJoined(config); - Assert.True(summary.HasCriticalValidationErrors); - Assert.Contains("You use JoinSummary options, but provided configurations cannot be joined", logger.GetLog()); - Assert.Contains("Orderer", logger.GetLog()); - } - - [Fact] - public void JoinedBenchmarksMustNotHaveDifferentExtraOrderers() - { - var benchmarks = new[] - { - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithExtraOrderer1)), - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithExtraOrderer2)) - }; - - var cases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - var validationErrors = - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(cases, null)) - .ToArray(); - - Assert.NotEmpty(validationErrors); - Assert.StartsWith("You use JoinSummary options, but provided configurations cannot be joined", validationErrors.Single().Message); - Assert.Contains("Orderer", validationErrors.Single().Message); - } - - [Fact] - public void JoinedBenchmarksMayHaveOneExtraOrderer() - { - var benchmarks = new[] - { - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithExtraOrderer1)), - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults1)) - }; - - var cases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - var validationErrors = - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(cases, null)) - .ToArray(); - - Assert.Empty(validationErrors); - } - - [Fact] - public void JoinedBenchmarksMayHaveDefaultOrderers() - { - var benchmarks = new[] - { - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults1)), - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults2)) - }; - - var cases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - var validationErrors = - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(cases, null)) - .ToArray(); - - Assert.Empty(validationErrors); - } - - [Fact] - public void JoinedBenchmarksMustNotHaveDifferentExtraSummaryStyles() - { - // Note: RatioStyle.Value would be the same as SummaryStyle.Default (SummaryStyle implements IEquatable). - ManualConfig config1 = DefaultConfig.Instance.WithSummaryStyle( - SummaryStyle.Default.WithRatioStyle(RatioStyle.Trend)); - ManualConfig config2 = DefaultConfig.Instance.WithSummaryStyle( - SummaryStyle.Default.WithRatioStyle(RatioStyle.Percentage)); - - var benchmarks = new[] - { - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults1), config1), - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults2), config2) - }; - - var cases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - var validationErrors = - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(cases, null)) - .ToArray(); - - Assert.NotEmpty(validationErrors); - Assert.StartsWith("You use JoinSummary options, but provided configurations cannot be joined", validationErrors.Single().Message); - Assert.Contains("SummaryStyle", validationErrors.Single().Message); - } - - [Fact] - public void JoinedBenchmarksMayHaveOneExtraSummaryStyle() - { - ManualConfig config = DefaultConfig.Instance.WithSummaryStyle( - SummaryStyle.Default.WithZeroMetricValuesInContent()); - var benchmarks = new[] - { - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults1), null), - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults2), config) - }; - - var cases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - var validationErrors = - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(cases, null)) - .ToArray(); - - Assert.Empty(validationErrors); - } - - [Orderer(SummaryOrderPolicy.Method)] - public class BenchmarkClassWithExtraOrderer1 - { - [Benchmark] - public void Foo() { } - } - - [Orderer(SummaryOrderPolicy.Method)] - public class BenchmarkClassWithExtraOrderer2 - { - [Benchmark] - public void Bar() { } - } - - public class BenchmarkClassWithDefaults1 - { - [Benchmark] - public void Baz() { } - } - - public class BenchmarkClassWithDefaults2 - { - [Benchmark] - public void Buzz() { } - } - } -} diff --git a/tests/BenchmarkDotNet.Tests/Validators/ExecutionValidatorTests.cs b/tests/BenchmarkDotNet.Tests/Validators/ExecutionValidatorTests.cs index 343bdc53dc..70195fd959 100644 --- a/tests/BenchmarkDotNet.Tests/Validators/ExecutionValidatorTests.cs +++ b/tests/BenchmarkDotNet.Tests/Validators/ExecutionValidatorTests.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Threading.Tasks; +using System.Threading.Tasks.Sources; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; @@ -563,5 +564,82 @@ public async ValueTask GlobalCleanup() [Benchmark] public void NonThrowing() { } } + + private class ValueTaskSource : IValueTaskSource, IValueTaskSource + { + private ManualResetValueTaskSourceCore _core; + + T IValueTaskSource.GetResult(short token) => _core.GetResult(token); + void IValueTaskSource.GetResult(short token) => _core.GetResult(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _core.GetStatus(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _core.GetStatus(token); + void IValueTaskSource.OnCompleted(Action continuation, object state, short token, ValueTaskSourceOnCompletedFlags flags) => _core.OnCompleted(continuation, state, token, flags); + void IValueTaskSource.OnCompleted(Action continuation, object state, short token, ValueTaskSourceOnCompletedFlags flags) => _core.OnCompleted(continuation, state, token, flags); + public void Reset() => _core.Reset(); + public short Token => _core.Version; + public void SetResult(T result) => _core.SetResult(result); + } + + [Fact] + public void AsyncValueTaskBackedByIValueTaskSourceIsAwaitedProperly() + { + var validationErrors = ExecutionValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(typeof(AsyncValueTaskSource))).ToList(); + + Assert.True(AsyncValueTaskSource.WasCalled); + Assert.Empty(validationErrors); + } + + public class AsyncValueTaskSource + { + private readonly ValueTaskSource valueTaskSource = new(); + + public static bool WasCalled; + + [GlobalSetup] + public ValueTask GlobalSetup() + { + valueTaskSource.Reset(); + Task.Delay(1).ContinueWith(_ => + { + WasCalled = true; + valueTaskSource.SetResult(true); + }); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + + [Benchmark] + public void NonThrowing() { } + } + + [Fact] + public void AsyncGenericValueTaskBackedByIValueTaskSourceIsAwaitedProperly() + { + var validationErrors = ExecutionValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(typeof(AsyncGenericValueTaskSource))).ToList(); + + Assert.True(AsyncGenericValueTaskSource.WasCalled); + Assert.Empty(validationErrors); + } + + public class AsyncGenericValueTaskSource + { + private readonly ValueTaskSource valueTaskSource = new(); + + public static bool WasCalled; + + [GlobalSetup] + public ValueTask GlobalSetup() + { + valueTaskSource.Reset(); + Task.Delay(1).ContinueWith(_ => + { + WasCalled = true; + valueTaskSource.SetResult(1); + }); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + + [Benchmark] + public void NonThrowing() { } + } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs b/tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs new file mode 100644 index 0000000000..9eac2dd6f2 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs @@ -0,0 +1,125 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Validators; +using System.Linq; +using Xunit; + +namespace BenchmarkDotNet.Tests.Validators; + +public class RuntimeValidatorTests +{ + [Fact] + public void SameRuntime_Should_Success() + { + // Arrange + var config = new TestConfig1().CreateImmutableConfig(); + var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config); + var parameters = new ValidationParameters(runInfo.BenchmarksCases, config); + + // Act + var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray(); + + // Assert + Assert.Empty(errors); + } + + [Fact] + public void NullRuntimeMixed_Should_Failed() + { + // Arrange + var config = new TestConfig2().CreateImmutableConfig(); + var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config); + var parameters = new ValidationParameters(runInfo.BenchmarksCases, config); + + // Act + var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray(); + + // Assert + { + var expectedMessage = "Job(Dry) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly."; + Assert.Contains(expectedMessage, errors); + } + { + var expectedMessage = "Job(Toolchain=.NET 10.0) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly."; + Assert.Contains(expectedMessage, errors); + } + } + + [Fact] + public void NotNullRuntimeOnly_Should_Success() + { + // Arrange + var config = new TestConfig3().CreateImmutableConfig(); + var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config); + var parameters = new ValidationParameters(runInfo.BenchmarksCases, config); + + // Act + var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray(); + + // Assert + Assert.Empty(errors); + } + + public class DummyBenchmark + { + [Benchmark] + public void Benchmark() + { + } + } + + // TestConfig that expicitly specify runtime. + private class TestConfig1 : ManualConfig + { + public TestConfig1() + { + var baseJob = Job.Dry; + + WithOption(ConfigOptions.DisableOptimizationsValidator, true); + AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray()); + + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80)); + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90)); + } + } + + // TestConfig that contains job that don't specify runtime. + private class TestConfig2 : ManualConfig + { + public TestConfig2() + { + var baseJob = Job.Dry; + + WithOption(ConfigOptions.DisableOptimizationsValidator, true); + AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray()); + + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80)); + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90) + .WithRuntime(CoreRuntime.Core90)); + + // Validate error message for auto generated jobid. + AddJob(Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0)); + } + } + + // TestConfig that expicitly specify runtime. + private class TestConfig3 : ManualConfig + { + public TestConfig3() + { + var baseJob = Job.Dry; + + WithOption(ConfigOptions.DisableOptimizationsValidator, true); + AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray()); + + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80) + .WithRuntime(CoreRuntime.Core80)); ; + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90) + .WithRuntime(CoreRuntime.Core90)); + } + } +} diff --git a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs index cb2b1a3878..79bfb5a1ae 100644 --- a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs +++ b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs @@ -8,5 +8,5 @@ public enum EnvRequirement FullFrameworkOnly, NonFullFramework, DotNetCoreOnly, - DotNetCore30Only + NeedsPrivilegedProcess } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs index e0510b92f6..22f131455d 100644 --- a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs +++ b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Runtime.InteropServices; +using System.Security.Principal; using BenchmarkDotNet.Jobs; using JetBrains.Annotations; using BdnRuntimeInformation = BenchmarkDotNet.Portability.RuntimeInformation; @@ -19,9 +20,19 @@ public static class EnvRequirementChecker EnvRequirement.FullFrameworkOnly => BdnRuntimeInformation.IsFullFramework ? null : "Full .NET Framework-only test", EnvRequirement.NonFullFramework => !BdnRuntimeInformation.IsFullFramework ? null : "Non-Full .NET Framework test", EnvRequirement.DotNetCoreOnly => BdnRuntimeInformation.IsNetCore ? null : ".NET/.NET Core-only test", - EnvRequirement.DotNetCore30Only => IsRuntime(RuntimeMoniker.NetCoreApp30) ? null : ".NET Core 3.0-only test", + EnvRequirement.NeedsPrivilegedProcess => IsPrivilegedProcess() ? null : "Needs authorization to perform security-relevant functions", _ => throw new ArgumentOutOfRangeException(nameof(requirement), requirement, "Unknown value") }; + private static bool IsPrivilegedProcess() + { +#if NET462 + using WindowsIdentity currentUser = WindowsIdentity.GetCurrent(); + return new WindowsPrincipal(currentUser).IsInRole(WindowsBuiltInRole.Administrator); +#else + return Environment.IsPrivilegedProcess; +#endif + } + private static bool IsRuntime(RuntimeMoniker moniker) => BdnRuntimeInformation.GetCurrentRuntime().RuntimeMoniker == moniker; } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/dotMemory/DotMemoryTests.cs b/tests/BenchmarkDotNet.Tests/dotMemory/DotMemoryTests.cs new file mode 100644 index 0000000000..d3aec8dd96 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/dotMemory/DotMemoryTests.cs @@ -0,0 +1,17 @@ +using System; +using BenchmarkDotNet.Diagnostics.dotMemory; +using BenchmarkDotNet.Jobs; +using Xunit; + +namespace BenchmarkDotNet.Tests.dotMemory; + +public class DotMemoryTests +{ + [Fact] + public void AllRuntimeMonikerAreKnown() + { + var diagnoser = new DotMemoryDiagnoser(); + foreach (RuntimeMoniker moniker in Enum.GetValues(typeof(RuntimeMoniker))) + diagnoser.IsSupported(moniker); // Just check that it doesn't throw exceptions + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/dotTrace/DotTraceTests.cs b/tests/BenchmarkDotNet.Tests/dotTrace/DotTraceTests.cs index a71cae2aef..751412fd55 100644 --- a/tests/BenchmarkDotNet.Tests/dotTrace/DotTraceTests.cs +++ b/tests/BenchmarkDotNet.Tests/dotTrace/DotTraceTests.cs @@ -3,15 +3,15 @@ using BenchmarkDotNet.Jobs; using Xunit; -namespace BenchmarkDotNet.Tests.dotTrace +namespace BenchmarkDotNet.Tests.dotTrace; + +public class DotTraceTests { - public class DotTraceTests + [Fact] + public void AllRuntimeMonikerAreKnown() { - [Fact] - public void AllRuntimeMonikerAreKnown() - { - foreach (RuntimeMoniker moniker in Enum.GetValues(typeof(RuntimeMoniker))) - DotTraceDiagnoser.IsSupported(moniker); // Just check that it doesn't throw exceptions - } + var diagnoser = new DotTraceDiagnoser(); + foreach (RuntimeMoniker moniker in Enum.GetValues(typeof(RuntimeMoniker))) + diagnoser.IsSupported(moniker); // Just check that it doesn't throw exceptions } } \ No newline at end of file diff --git a/tests/runCoreTests.cmd b/tests/runCoreTests.cmd index 93c5c0f258..137ea952d9 100644 --- a/tests/runCoreTests.cmd +++ b/tests/runCoreTests.cmd @@ -12,24 +12,24 @@ if NOT %ERRORLEVEL% == 0 ( ) echo ----------------------------- -echo Running Core 2.1 Unit tests +echo Running Core 8.0 Unit tests echo ----------------------------- -call dotnet test "BenchmarkDotNet.Tests\BenchmarkDotNet.Tests.csproj" --configuration Release --framework netcoreapp2.1 +call dotnet test "BenchmarkDotNet.Tests\BenchmarkDotNet.Tests.csproj" --configuration Release --framework net8.0 if NOT %ERRORLEVEL% == 0 ( - echo CORE 2.1 Unit tests has failed + echo Core 8.0 Unit tests has failed goto end ) echo ----------------------------- -echo Running Core 2.1 Integration tests +echo Running 8.0 Integration tests echo ----------------------------- -call dotnet test "BenchmarkDotNet.IntegrationTests\BenchmarkDotNet.IntegrationTests.csproj" --configuration Release --framework netcoreapp2.1 +call dotnet test "BenchmarkDotNet.IntegrationTests\BenchmarkDotNet.IntegrationTests.csproj" --configuration Release --framework net8.0 if NOT %ERRORLEVEL% == 0 ( - echo CORE 2.1 Integration tests has failed + echo Core 8.0 Integration tests has failed goto end ) diff --git a/tests/runCoreTests.sh b/tests/runCoreTests.sh index 4e3ad59e6a..1b5bb28fa3 100755 --- a/tests/runCoreTests.sh +++ b/tests/runCoreTests.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -dotnet test BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj --configuration Release --framework netcoreapp2.1 2>&1 | tee tests.log -dotnet test BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj --configuration Release --framework netcoreapp2.1 2>&1 | tee integration-tests.log +dotnet test BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj --configuration Release --framework net8.0 2>&1 | tee tests.log +dotnet test BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj --configuration Release --framework net8.0 2>&1 | tee integration-tests.log