split perf jobs into test categories

eng/common/performance/crossgen_perf.proj

@@ -2,7 +2,6 @@
 
   <PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
     <Python>py -3</Python>
-    <BaselineCoreRun>%HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe</BaselineCoreRun>
     <HelixPreCommands>$(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD%</HelixPreCommands>
     <ArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts</ArtifactsDirectory>
     <BaseDirectory>$HELIX_CORRELATION_PAYLOAD</BaseDirectory>
@@ -14,7 +13,6 @@
     <CliArguments>--dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP</CliArguments>
     <Python>python3</Python>
     <CoreRun>$(BaseDirectory)/Core_Root/corerun</CoreRun>
-    <BaselineCoreRun>$(BaseDirectory)/Baseline_Core_Root/corerun</BaselineCoreRun>
     <HelixPreCommands>$(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh</HelixPreCommands>
     <ArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts</ArtifactsDirectory>
     <BaselineArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts_Baseline</BaselineArtifactsDirectory>

eng/pipelines/coreclr/perf.yml

@@ -43,6 +43,7 @@ jobs:
 #
 - template: /eng/pipelines/common/checkout-job.yml
 
+# build coreclr and libraries
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
     jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml
@@ -54,6 +55,7 @@ jobs:
     jobParameters:
       testGroup: perf
 
+# build mono
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
     jobTemplate: /eng/pipelines/mono/templates/build-job.yml
@@ -62,6 +64,7 @@ jobs:
     platforms:
     - Linux_x64
 
+# run mono perf job
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
     jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml
@@ -73,7 +76,10 @@ jobs:
       testGroup: perf
       liveLibrariesBuildConfig: Release
       runtimeType: mono
+      projectFile: microbenchmarks.proj
+      testCategory: microbenchmarks
 
+# run mono interpreter job (private CI only)
 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
   - template: /eng/pipelines/common/platform-matrix.yml
     parameters:
@@ -87,7 +93,10 @@ jobs:
         liveLibrariesBuildConfig: Release
         runtimeType: mono
         codeGenType: 'Interpreter'
+        projectFile: microbenchmarks.proj
+        testCategory: microbenchmarks
 
+# run coreclr microbenchmarks perf job
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
     jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml
@@ -100,3 +109,22 @@ jobs:
     jobParameters:
       testGroup: perf
       liveLibrariesBuildConfig: Release
+      projectFile: microbenchmarks.proj
+      testCategory: microbenchmarks
+
+# run coreclr crossgen perf job
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml
+    buildConfig: release
+    runtimeFlavor: coreclr
+    platforms:
+    #- Linux_x64
+    - Windows_NT_x64
+    - Windows_NT_x86
+    jobParameters:
+      testGroup: perf
+      liveLibrariesBuildConfig: Release
+      projectFile: crossgen_perf.proj
+      testCategory: crossgen
+

eng/pipelines/coreclr/templates/perf-job.yml

@@ -11,6 +11,8 @@ parameters:
   runtimeType: 'coreclr'
   pool: ''
   codeGenType: 'JIT'
+  projetFile: ''
+  testCategory: ''
 
 ### Perf job
 
@@ -21,8 +23,8 @@ jobs:
 - template: run-performance-job.yml
   parameters:
     # Compute job name from template parameters
-    jobName: ${{ format('perfbuild_{0}{1}_{2}_{3}_{4}_{5}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType) }}
-    displayName: ${{ format('Performance {0}{1} {2} {3} {4} {5}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType) }}
+    jobName: ${{ format('perfbuild_{0}{1}_{2}_{3}_{4}_{5}_{6}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType, parameters.testCategory) }}
+    displayName: ${{ format('Performance {0}{1} {2} {3} {4} {5} {6}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType, parameters.testCategory) }}
     pool: ${{ parameters.pool }}
     buildConfig: ${{ parameters.buildConfig }}
     archType: ${{ parameters.archType }}
@@ -32,6 +34,7 @@ jobs:
     liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
     runtimeType: ${{ parameters.runtimeType }}
     codeGenType: ${{ parameters.codeGenType }}
+    projectFile: ${{ parameters.projectFile }}
     # Test job depends on the corresponding build job
     dependsOn:
     - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
@@ -40,15 +43,16 @@ jobs:
     - ${{ if eq(parameters.runtimeType, 'mono') }}:
       - ${{ format('mono_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
 
-    ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
-      ${{ if eq(parameters.runtimeType, 'mono') }}:
-        extraSetupParameters: -Architecture ${{ parameters.archType }} -MonoDotnet $(Build.SourcesDirectory)\.dotnet-mono -Kind micro_mono
-      ${{ if ne(parameters.runtimeType, 'mono') }}:
-        extraSetupParameters: -CoreRootDirectory $(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.Release\Tests\Core_Root -Architecture ${{ parameters.archType }}
-    ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
-      ${{ if eq(parameters.runtimeType, 'mono') }}:
+    ${{ if eq(parameters.testCategory, 'microbenchmarks') }}:
+      ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
+        ${{ if eq(parameters.runtimeType, 'mono') }}:
+          extraSetupParameters: -Architecture ${{ parameters.archType }} -MonoDotnet $(Build.SourcesDirectory)\.dotnet-mono -Kind micro_mono
+        ${{ if ne(parameters.runtimeType, 'mono') }}:
+          extraSetupParameters: -CoreRootDirectory $(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.Release\Tests\Core_Root -Architecture ${{ parameters.archType }}
+      ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
+        ${{ if eq(parameters.runtimeType, 'mono') }}:
         extraSetupParameters: --architecture ${{ parameters.archType }} --monodotnet $(Build.SourcesDirectory)/.dotnet-mono --kind micro_mono
-      ${{ if ne(parameters.runtimeType, 'mono') }}:
+        ${{ if ne(parameters.runtimeType, 'mono') }}:
         extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }}
 
     variables: ${{ parameters.variables }}

eng/pipelines/coreclr/templates/run-performance-job.yml

@@ -18,7 +18,7 @@ parameters:
   liveLibrariesBuildConfig: ''    # optional -- live-live libraries configuration to use for the run
   runtimeType: 'coreclr'          # optional -- Sets the runtime as coreclr or mono
   codeGenType: 'JIT'              # optional -- Decides on the codegen technology if running on mono
-  projectFile:                    # required -- project file to build helix workitems
+  projectFile: ''                 # required -- project file to build helix workitems
 
 jobs:
 - template: xplat-pipeline-job.yml
@@ -134,10 +134,10 @@ jobs:
         WorkItemDirectory: '$(WorkItemDirectory)' # WorkItemDirectory can not be empty, so we send it some docs to keep it happy
         CorrelationPayloadDirectory: '$(PayloadDirectory)' # it gets checked out to a folder with shorter path than WorkItemDirectory so we can avoid file name too long exceptions
         ProjectFile: ${{ parameters.projectFile }}
-        - task: PublishPipelineArtifact@1
+    - task: PublishPipelineArtifact@1
       displayName: Publish Logs
       inputs:
         targetPath: $(Build.SourcesDirectory)/artifacts/log
-        artifactName: 'Performance_Run_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.runtimeType }}_${{ parameters.codeGenType }}'
+        artifactName: 'Performance_Run_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.runtimeType }}_${{ parameters.codeGenType }}_${{ parameters.testCategory }}'
       continueOnError: true
       condition: always()