Separate running PAL tests out to their own jobs/steps.

eng/pipelines/common/templates/runtimes/run-test-job.yml

@@ -345,11 +345,7 @@ jobs:
 
         timeoutPerTestInMinutes: $(timeoutPerTestInMinutes)
         timeoutPerTestCollectionInMinutes: $(timeoutPerTestCollectionInMinutes)
-
         runCrossGen2: ${{ eq(parameters.readyToRun, true) }}
-        ${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.runtimeFlavor, 'coreclr')) }}:
-          runPALTestsDir: '$(coreClrProductRootFolderPath)/paltests'
-
         compositeBuildMode: ${{ parameters.compositeBuildMode }}
         runInUnloadableContext: ${{ parameters.runInUnloadableContext }}
         tieringTest: ${{ parameters.tieringTest }}

eng/pipelines/common/templates/runtimes/send-to-helix-step.yml

@@ -17,7 +17,6 @@ parameters:
   timeoutPerTestCollectionInMinutes: ''
   timeoutPerTestInMinutes: ''
   runCrossGen2: ''
-  runPALTestsDir: ''
   compositeBuildMode: false
   helixProjectArguments: ''
   runInUnloadableContext: ''
@@ -60,7 +59,6 @@ steps:
       _LongRunningGcTests: ${{ parameters.longRunningGcTests }}
       _GcSimulatorTests: ${{ parameters.gcSimulatorTests }}
       _Scenarios: ${{ join(',', parameters.scenarios) }}
-      _PALTestsDir: ${{ parameters.runPALTestsDir }}
       _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }}
       _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
       RuntimeFlavor: ${{ parameters.runtimeFlavor }}

eng/pipelines/coreclr/ci.yml

@@ -165,3 +165,23 @@ extends:
           platforms:
           - linux_x64
           - windows_x64
+
+      #
+      # PAL Tests
+      #
+      - template: /eng/pipelines/common/platform-matrix.yml
+        parameters:
+          helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+          jobTemplate: /eng/pipelines/common/global-build-job.yml
+          buildConfig: Debug
+          platforms:
+          - linux_x64
+          - linux_musl_x64
+          - linux_arm64
+          - linux_musl_arm64
+          - osx_x64
+          - osx_arm64
+          jobParameters:
+            buildArgs: -s clr.paltests+clr.paltestlist
+            nameSuffix: PALTests
+            extraStepsTemplate: /eng/pipelines/coreclr/templates/run-paltests-step.yml

eng/pipelines/coreclr/templates/build-job.yml

@@ -92,7 +92,7 @@ jobs:
       value: ''
     - ${{ if ne(parameters.testGroup, 'innerloop') }}:
       - name: clrRuntimeComponentsBuildArg
-        value: '-component runtime -component alljits -component paltests -component nativeaot -component spmi '
+        value: '-component runtime -component alljits -component nativeaot -component spmi '
 
     - name: pgoInstrumentArg
       value: ''
@@ -172,7 +172,7 @@ jobs:
         displayName: Disk Usage after Build
 
     # Build CoreCLR Managed Components
-    - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+clr.paltestlist $(crossArg) $(compilerArg) -arch $(archType) $(osArg) -c $(buildConfig) $(pgoInstrumentArg) $(officialBuildIdArg) -ci
+    - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages $(crossArg) $(compilerArg) -arch $(archType) $(osArg) -c $(buildConfig) $(pgoInstrumentArg) $(officialBuildIdArg) -ci
       displayName: Build managed product components and packages
 
     # Build native test components

eng/pipelines/coreclr/templates/run-paltests-step.yml

@@ -0,0 +1,23 @@
+parameters:
+  continueOnError: 'false'        # optional -- determines whether to continue the build if the step errors
+  helixQueues: ''                 # required -- Helix queues
+  buildConfig: ''                 # required -- build configuration
+  archType: ''                    # required -- targeting CPU architecture
+  osGroup: ''                     # required -- operating system for the job
+  osSubgroup: ''                  # optional -- operating system subgroup
+  dependOnEvaluatePaths: false
+
+steps:
+  - template: /eng/pipelines/common/templates/runtimes/send-to-helix-step.yml
+    parameters:
+      displayName: 'Send job to Helix'
+      helixBuild: $(Build.BuildNumber)
+      helixSource: $(_HelixSource)
+      helixType: 'build/tests/'
+      helixQueues: ${{ parameters.helixQueues }}
+      creator: dotnet-bot
+      helixProjectArguments: '$(Build.SourcesDirectory)/src/coreclr/scripts/paltests.proj'
+      BuildConfig: ${{ parameters.buildConfig }}
+      osGroup: ${{ parameters.osGroup }}
+      archType: ${{ parameters.archType }}
+      shouldContinueOnError: ${{ parameters.continueOnError }}

src/coreclr/scripts/paltests.proj

@@ -0,0 +1,44 @@
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+  <PropertyGroup>
+    <EnableXUnitReporter>true</EnableXUnitReporter>
+    <Creator>$(_Creator)</Creator>
+    <HelixAccessToken>$(_HelixAccessToken)</HelixAccessToken>
+    <HelixBuild>$(_HelixBuild)</HelixBuild>
+    <HelixSource>$(_HelixSource)</HelixSource>
+    <HelixTargetQueues>$(_HelixTargetQueues)</HelixTargetQueues>
+    <HelixType>$(_HelixType)</HelixType>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PalTest Include="$(CoreClrArtifactsPath)/paltests/**/*" />
+    <PalTestArchive Include="$(OutputPath)paltests.tar.gz" />
+  </ItemGroup>
+
+  <Target
+    Name="PrepareArchive"
+    Inputs="@(PalTest)"
+    Outputs="@(PalTestArchive)">
+    <MakeDir Directories="$(OutputPath)" Condition="!Exists('$(OutputPath)')"/>
+    <!--
+      We need to preserve the permission bits on the PAL tests.
+      Helix's default archive of Zip files does not preserve these bits,
+      so we'll manually make a tarball and upload that instead.
+     -->
+    <Exec
+          WorkingDirectory="$(CoreCLRArtifactsPath)/paltests"
+          Command="tar -czvf $(OutputPath)paltests.tar.gz `ls -A`"/>
+  </Target>
+
+  <Target
+    Name="AddWorkItem"
+    BeforeTargets="CoreTest"
+    DependsOnTargets="PrepareArchive">
+    <ItemGroup>
+      <HelixWorkItem Include="PALTests">
+        <PayloadArchive>@(PalTestArchive)</PayloadArchive>
+        <Command>./runpaltestshelix.sh</Command>
+        <Timeout>0:10</Timeout>
+      </HelixWorkItem>
+    </ItemGroup>
+  </Target>
+</Project>

src/tests/Common/helixpublishwitharcade.proj

@@ -47,7 +47,6 @@
     <_RuntimeVariant></_RuntimeVariant>
     <BundledNETCoreAppPackageVersion>BundledNETCoreAppPackageVersion</BundledNETCoreAppPackageVersion>
     <HelixRuntimeRid></HelixRuntimeRid>
-    <_PALTestsDir></_PALTestsDir>
     <_SuperPmiCollect>false</_SuperPmiCollect>
   </PropertyGroup>
   <Target Name="printItems">
@@ -107,7 +106,6 @@
         RuntimeVariant=$(_RuntimeVariant);
         BundledNETCoreAppPackageVersion=$(BundledNETCoreAppPackageVersion);
         HelixRuntimeRid=$(HelixRuntimeRid);
-        PALTestsDir=$(_PALTestsDir);
         SuperPmiCollect=$(_SuperPmiCollect)
       </_PropertiesToPass>
 
@@ -134,7 +132,6 @@
 
     <MSBuild Projects="$(MSBuildProjectFile)" Targets="PrepareCorrelationPayloadDirectory" Properties="$(_PropertiesToPass)" />
     <MSBuild Projects="$(MSBuildProjectFile)" Targets="PreparePayloadDirectories" Properties="$(_PropertiesToPass)" StopOnFirstFailure="true" />
-    <MSBuild Projects="$(MSBuildProjectFile)" Targets="PreparePALTestArchive" Properties="$(_PropertiesToPass)" StopOnFirstFailure="true" />
 
     <ItemGroup>
       <_Scenarios Include="$(_Scenarios.Split(','))" />
@@ -326,12 +323,6 @@
     <MSBuild Projects="@(_ProjectsToBuild)" Targets="CreateTestEnvFile" StopOnFirstFailure="true" />
   </Target>
 
-  <Target Name="PreparePALTestArchive">
-    <Exec Condition="'$(PALTestsDir)' != '' and '$(TestWrapperTargetsWindows)' != 'true'"
-          WorkingDirectory="$(PALTestsDir)"
-          Command="tar -czvf $(LegacyPayloadsRootDirectory)paltests.tar.gz `ls -A`"/>
-  </Target>
-
   <Target Name="PreparePayloadDirectories" DependsOnTargets="PrepareMergedTestPayloadDirectories;PrepareLegacyPayloadDirectories" />
 
   <Target Name="PrepareLegacyPayloadDirectories" DependsOnTargets="PrepareLegacyPayloadDirectory">
@@ -933,12 +924,6 @@
       <TestTarget>$(AppleTestTarget)</TestTarget>
       <TestTimeout Condition=" '$(TimeoutPerTestCollectionInMinutes)' != '' ">$([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes)))</TestTimeout>
     </XHarnessAppBundleToTest>
-
-    <HelixWorkItem Condition="'$(PALTestsDir)' != '' and '$(TestWrapperTargetsWindows)' != 'true'" Include="PALTests">
-      <PayloadArchive>$(LegacyPayloadsRootDirectory)paltests.tar.gz</PayloadArchive>
-      <Command>$(_WorkaroundForNuGetMigrationsForPrepending) ./runpaltestshelix.sh</Command>
-      <Timeout Condition=" '$(TimeoutPerTestCollectionInMinutes)' != '' ">$([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes)))</Timeout>
-    </HelixWorkItem>
   </ItemGroup>
 
   <Import Sdk="Microsoft.DotNet.Helix.Sdk" Project="Sdk.targets" Condition=" '$(UsesHelixSdk)' == 'true' " />