Add dotnet cli to runtime test Helix jobs (#35426)

* Currently managed tools such as crossgen2 and XUnit are run against the built runtime which is slow on Debug builds, and can obscure errors when the XUnit test harness fails due to an introduced runtime bug.
* Add `xunit.console.runtimeconfig.dev.json` which allows `xunit.console` to use the repo-local dotnet.cmd on dev boxes and the same version installed on the path in Helix.
* When running crossgen2 during test execution, support both local dev and Helix scenario when deciding which dotnet to run. On Helix, simply use `dotnet` which assumes a compatible dotnet runtime is in the path. Locally, tests are run with `runtest.cmd|sh` which sets `__TestDotNetCmd` to the repo-local dotnet script. This preserves the characteristic that no machine-wide 5.0 dotnet runtime must be installed for the runtime tests.
* Update batch scripting to also use `dotnet.cmd|sh` when running Crossgen2 replacing corerun.exe.
* crossgen2's `runtimes` folder is not getting copied to `CORE_ROOT` which causes the runtime host to abort the launch on the Unix CI VMs since crossgen.deps.json refers to files in that subfolder. Adjust the `CORE_ROOT` pruning in `Directory.Build.targets` to include subfolders for the two tools that need it.
* Improve XUnit test boilerplate. Printing `Exception.Message` doesn't include stack trace. Use `ToString()` instead.
* Import notargets sdk in `helixpublicwitharcade.proj`. It doesn't use the official sdk so `BundledNETCoreAppPackageVersion` wasn't set. Import the `Microsoft.Build.NoTargets` sdk so we can find the bundled runtime package version.

Author: nattress

eng/Versions.props

@@ -63,7 +63,7 @@
     <MicrosoftDotNetGenAPIVersion>5.0.0-beta.20261.9</MicrosoftDotNetGenAPIVersion>
     <MicrosoftDotNetGenFacadesVersion>5.0.0-beta.20261.9</MicrosoftDotNetGenFacadesVersion>
     <MicrosoftDotNetXUnitExtensionsVersion>5.0.0-beta.20261.9</MicrosoftDotNetXUnitExtensionsVersion>
-    <MicrosoftDotNetXUnitConsoleRunnerVersion>2.5.1-beta.20261.9</MicrosoftDotNetXUnitConsoleRunnerVersion>
+    <MicrosoftDotNetXUnitConsoleRunnerVersion>2.5.1-beta.20264.3</MicrosoftDotNetXUnitConsoleRunnerVersion>
     <MicrosoftDotNetBuildTasksPackagingVersion>5.0.0-beta.20261.9</MicrosoftDotNetBuildTasksPackagingVersion>
     <MicrosoftDotNetRemoteExecutorVersion>5.0.0-beta.20261.9</MicrosoftDotNetRemoteExecutorVersion>
     <MicrosoftDotNetVersionToolsTasksVersion>5.0.0-beta.20261.9</MicrosoftDotNetVersionToolsTasksVersion>

src/coreclr/build-test.cmd

@@ -652,15 +652,17 @@ for %%F in ("%CORE_ROOT%\System.*.dll";"%CORE_ROOT%\Microsoft.*.dll";%CORE_ROOT%
     )))))
 )
 
-echo Composite response line^: %__CompositeResponseFile%
-type "%__CompositeResponseFile%"
+if defined __CompositeBuildMode (
+    echo Composite response line^: %__CompositeResponseFile%
+    type "%__CompositeResponseFile%"
+)
 
 if defined __CompositeBuildMode (
-    set __CompositeCommandLine="%CORE_ROOT%\corerun"
+    set __CompositeCommandLine="%__RepoRootDir%\dotnet.cmd"
     set __CompositeCommandLine=!__CompositeCommandLine! "%CORE_ROOT%\crossgen2\crossgen2.dll"
     set __CompositeCommandLine=!__CompositeCommandLine! "@%__CompositeResponseFile%"
     echo Building composite R2R framework^: !__CompositeCommandLine!
-    !__CompositeCommandLine!
+    call !__CompositeCommandLine!
     set __FailedToPrecompile=!ERRORLEVEL!
     copy /Y "!__CompositeOutputDir!\*.*" "!CORE_ROOT!\"
 )
@@ -684,7 +686,7 @@ if /i "%__BuildArch%" == "arm64" ( set __CrossgenExe="%__BinDir%\x64\crossgen.ex
 set __CrossgenExe=%__CrossgenExe%
 
 if defined __DoCrossgen2 (
-    set __CrossgenExe="%CORE_ROOT%\corerun" "%__BinDir%\crossgen2\crossgen2.dll"
+    set __CrossgenExe="%__RepoRootDir%\dotnet.cmd" "%CORE_ROOT%\crossgen2\crossgen2.dll"
 )
 
 REM Intentionally avoid using the .dll extension to prevent
@@ -694,12 +696,14 @@ set __CrossgenCmd=
 
 if defined __DoCrossgen (
     set __CrossgenCmd=!__CrossgenExe! /Platform_Assemblies_Paths "!CORE_ROOT!" /in !AssemblyPath! /out !__CrossgenOutputFile!
+    echo !__CrossgenCmd!
+    !__CrossgenCmd!
 ) else (
     set __CrossgenCmd=!__CrossgenExe! -r:"!CORE_ROOT!\System.*.dll" -r:"!CORE_ROOT!\Microsoft.*.dll" -r:"!CORE_ROOT!\mscorlib.dll" -r:"!CORE_ROOT!\netstandard.dll" -O --inputbubble --out:!__CrossgenOutputFile! !AssemblyPath!
+    echo !__CrossgenCmd!
+    call !__CrossgenCmd!
 )
 
-echo %__CrossgenCmd%
-%__CrossgenCmd%
 set /a __exitCode = !errorlevel!
 
 set /a "%~3+=1"

src/coreclr/build-test.sh

@@ -180,8 +180,8 @@ precompile_coreroot_fx()
 
     local totalPrecompiled=0
     local failedToPrecompile=0
-    local compositeCommandLine="$overlayDir/corerun"
-    compositeCommandLine+=" ${__BinDir}/crossgen2/crossgen2.dll"
+    local compositeCommandLine="${__DotNetCli}"
+    compositeCommandLine+=" $__BinDir/crossgen2/crossgen2.dll"
     compositeCommandLine+=" --composite"
     compositeCommandLine+=" -O"
     compositeCommandLine+=" --out:$outputDir/framework-r2r.dll"
@@ -206,7 +206,7 @@ precompile_coreroot_fx()
         fi
 
         if [[ "$__DoCrossgen2" != 0 ]]; then
-            commandLine="$overlayDir/corerun $overlayDir/crossgen2/crossgen2.dll $crossgen2References -O --inputbubble --out $outputDir/$(basename $filename) $filename"
+            commandLine="${__DotNetCli} $overlayDir/crossgen2/crossgen2.dll $crossgen2References -O --inputbubble --out $outputDir/$(basename $filename) $filename"
         fi
 
         echo Precompiling "$filename"

src/coreclr/tests/helixpublishwitharcade.proj

@@ -5,7 +5,7 @@
        to send test jobs to helix. -->
 
   <Import Sdk="Microsoft.DotNet.Helix.Sdk" Project="Sdk.props" Condition=" '$(UsesHelixSdk)' == 'true' " />
-  <Import Project="Directory.Build.props" Condition=" '$(UsesHelixSdk)' != 'true' "/>
+  <Import Sdk="Microsoft.Build.NoTargets" Project="Sdk.props" Condition=" '$(UsesHelixSdk)' != 'true' " />
 
   <!-- This target runs once and creates several instances of this project (one for each scenario)
        that will run in parallel. -->
@@ -33,10 +33,14 @@
         RunInUnloadableContext=$(_RunInUnloadableContext);
         TimeoutPerTestCollectionInMinutes=$(_TimeoutPerTestCollectionInMinutes);
         TimeoutPerTestInMinutes=$(_TimeoutPerTestInMinutes);
-        RuntimeMode=$(_RuntimeMode)
+        RuntimeMode=$(_RuntimeMode);
+        BundledNETCoreAppPackageVersion=$(BundledNETCoreAppPackageVersion)
       </_PropertiesToPass>
     </PropertyGroup>
 
+    <Message Text="DotNetCliVersion: $(DotNetCliVersion)" Importance="High" />
+    <Message Text="DotNetCliPackageType: $(DotNetCliPackageType)" Importance="High" />
+    <Message Text="HelixRuntimeRid: $(HelixRuntimeRid)" Importance="High" />
     <Error Condition="'$(_Scenarios)' == ''" Text="_Scenarios not set" />
 
     <MSBuild Projects="$(MSBuildProjectFile)" Targets="PrepareCorrelationPayloadDirectory" />
@@ -59,12 +63,24 @@
     <MSBuild Projects="@(_ProjectsToBuild)" Targets="Test" BuildInParallel="$(_BuildInParallel)" StopOnFirstFailure="false" Properties="UsesHelixSdk=true" />
   </Target>
 
+  <!-- Choose a suitable runtime RID for Helix to restore the dotnet cli -->
+  <PropertyGroup>
+    <HelixRuntimeRid Condition="'$(TargetOS)' == 'Windows_NT'">win-$(TargetArchitecture)</HelixRuntimeRid>
+    <HelixRuntimeRid Condition="'$(TargetOS)' == 'OSX'">osx-$(TargetArchitecture)</HelixRuntimeRid>
+    <HelixRuntimeRid Condition="'$(TargetOS)' == 'Linux' or '$(TargetOS)' == 'Linux_musl'">linux-$(TargetArchitecture)</HelixRuntimeRid>
+  </PropertyGroup>
+
   <PropertyGroup>
     <BinDir>$([MSBuild]::NormalizeDirectory($(TestWorkingDir)))</BinDir>
     <CoreRootDirectory>$(BinDir)Tests\Core_Root\</CoreRootDirectory>
     <PayloadsRootDirectory>$(BinDir)Payloads\</PayloadsRootDirectory>
     <TestEnvFileName Condition=" '$(TargetsWindows)' == 'true' ">SetStressModes_$(Scenario).cmd</TestEnvFileName>
     <TestEnvFileName Condition=" '$(TargetsWindows)' != 'true' ">SetStressModes_$(Scenario).sh</TestEnvFileName>
+
+    <IncludeDotNetCli>true</IncludeDotNetCli>
+    <DotNetCliPackageType>runtime</DotNetCliPackageType>
+    <DotNetCliVersion>$(BundledNETCoreAppPackageVersion)</DotNetCliVersion>
+    <DotNetCliRuntime>$(HelixRuntimeRid)</DotNetCliRuntime>
   </PropertyGroup>
 
   <Import Project="testgrouping.proj" />
@@ -236,12 +252,10 @@
   </PropertyGroup>
 
   <PropertyGroup Condition=" '$(TargetsWindows)' == 'true' ">
-    <CoreRun>%CORE_ROOT%\CoreRun.exe</CoreRun>
     <XUnitRunnerDll>%CORE_ROOT%\xunit.console.dll</XUnitRunnerDll>
   </PropertyGroup>
 
   <PropertyGroup Condition=" '$(TargetsWindows)' != 'true' ">
-    <CoreRun>$CORE_ROOT/corerun</CoreRun>
     <XUnitRunnerDll>$CORE_ROOT/xunit.console.dll</XUnitRunnerDll>
   </PropertyGroup>
 
@@ -263,12 +277,13 @@
 
     <HelixWorkItem Include="@(Payloads->Metadata('PayloadGroup'))">
       <PayloadDirectory>%(PayloadDirectory)</PayloadDirectory>
-      <Command>$(CoreRun) $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs)</Command>
-      <Command Condition=" '%(TestGroup)' != '' ">$(CoreRun) $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) -trait TestGroup=%(TestGroup)</Command>
+      <Command>dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs)</Command>
+      <Command Condition=" '%(TestGroup)' != '' ">dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) -trait TestGroup=%(TestGroup)</Command>
       <Timeout Condition=" '$(TimeoutPerTestCollectionInMinutes)' != '' ">$([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes)))</Timeout>
     </HelixWorkItem>
   </ItemGroup>
 
   <Import Sdk="Microsoft.DotNet.Helix.Sdk" Project="Sdk.targets" Condition=" '$(UsesHelixSdk)' == 'true' " />
+  <Import Sdk="Microsoft.Build.NoTargets" Project="Sdk.targets" Condition=" '$(UsesHelixSdk)' != 'true' " />
 
 </Project>

src/coreclr/tests/runtest.py

@@ -934,6 +934,9 @@ def run_tests(args,
     print("Setting CORE_ROOT=%s" % args.core_root)
     os.environ["CORE_ROOT"] = args.core_root
 
+    # Set __TestDotNetCmd so tests which need to run dotnet can use the repo-local script on dev boxes
+    os.environ["__TestDotNetCmd"] = args.dotnetcli_script_path
+
     # Set test env script path if it is set.
     if test_env_script_path is not None:
         print("Setting __TestEnv=%s" % test_env_script_path)

src/coreclr/tests/src/CLRTest.CrossGen.targets

@@ -83,7 +83,12 @@ if [ ! -z ${RunCrossGen2+x} ]%3B then
     rm $__ResponseFile
 
     __Command=$_DebuggerFullPath
-    __Command+=" $CORE_ROOT/corerun"
+    # Tests run locally need __TestDotNetCmd (set by runtest.py) or a compatible 5.0 dotnet runtime in the path
+    if [ ! -z ${__TestDotNetCmd+x} ] %3B then
+        __Command+=" $__TestDotNetCmd"
+    else
+        __Command+=" dotnet"
+    fi
     __Command+=" $CORE_ROOT/crossgen2/crossgen2.dll"
     __Command+=" @$__ResponseFile"
     __Command+=" $ExtraCrossGen2Args"
@@ -178,7 +183,12 @@ if defined RunCrossGen2 (
     del /Q !__ResponseFile!
 
     set __Command=!_DebuggerFullPath!
-    set __Command=!__Command! "!CORE_ROOT!\CoreRun.exe"
+    REM Tests run locally need __TestDotNetCmd (set by runtest.py) or a compatible 5.0 dotnet runtime in the path
+    if defined __TestDotNetCmd (
+        set __Command=!__Command! "!__TestDotNetCmd!"
+    ) else (
+        set __Command=!__Command! "dotnet"
+    )
     set __Command=!__Command! "!CORE_ROOT!\crossgen2\crossgen2.dll"
     set __Command=!__Command! @"!__ResponseFile!"
     set __Command=!__Command! !ExtraCrossGen2Args!

src/coreclr/tests/src/Common/Directory.Build.targets

@@ -58,26 +58,37 @@
       <RunTimeArtifactsIncludeFolders Include="IL/" />
 
       <!-- Used for Crossgen2 R2R tests -->
-      <RunTimeArtifactsIncludeFolders Include="crossgen2/" />
+      <RunTimeArtifactsIncludeFolders Include="crossgen2/">
+        <IncludeSubFolders>True</IncludeSubFolders>
+      </RunTimeArtifactsIncludeFolders>
 
       <!-- Used for capturing symbolic stack traces using Watson -->
       <RunTimeArtifactsIncludeFolders Include="PDB/" />
 
       <!-- Used by the coreroot_determinism test -->
-      <RunTimeArtifactsIncludeFolders Include="R2RTest/" />
+      <RunTimeArtifactsIncludeFolders Include="R2RTest/">
+        <IncludeSubFolders>True</IncludeSubFolders>
+      </RunTimeArtifactsIncludeFolders>
     </ItemGroup>
 
     <ItemGroup>
       <!-- Add binary dependencies to copy-local items -->
       <RunTimeDependencyCopyLocal
+          Condition="'%(RuntimeArtifactsIncludeFolders.IncludeSubFolders)' != 'True'"
           Include="$(CoreCLRArtifactsPath)%(RunTimeArtifactsIncludeFolders.Identity)*"
           Exclude="@(RunTimeArtifactsExcludeFiles -> '$(CoreCLRArtifactsPath)%(Identity)')"
           TargetDir="%(RunTimeArtifactsIncludeFolders.Identity)" />
+
+      <RunTimeDependencyCopyLocal
+          Condition="'%(RuntimeArtifactsIncludeFolders.IncludeSubFolders)' == 'True'"
+          Include="$(CoreCLRArtifactsPath)%(RunTimeArtifactsIncludeFolders.Identity)**/*"
+          Exclude="@(RunTimeArtifactsExcludeFiles -> '$(CoreCLRArtifactsPath)%(Identity)')"
+          TargetDir="%(RunTimeArtifactsIncludeFolders.Identity)" />
     </ItemGroup>
 
     <Copy
       SourceFiles="@(RunTimeDependencyCopyLocal)"
-      DestinationFiles="@(RunTimeDependencyCopyLocal -> '$(CORE_ROOT)/%(TargetDir)%(Filename)%(Extension)')"
+      DestinationFiles="@(RunTimeDependencyCopyLocal -> '$(CORE_ROOT)/%(TargetDir)%(RecursiveDir)%(Filename)%(Extension)')"
       SkipUnchangedFiles="$(SkipCopyUnchangedFiles)"
       OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
       Retries="$(CopyRetryCount)"

src/coreclr/tests/src/runtest.proj

@@ -291,7 +291,7 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\","").
 
                 if (infraEx != null)
                 {
-                    Assert.True(false, "Test Infrastructure Failure: " + infraEx.Message)%3B
+                    Assert.True(false, "Test Infrastructure Failure: " + infraEx.ToString())%3B
                 }
                 else
                 {
@@ -304,7 +304,7 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\","").
                     catch (Exception ex)
                     {
                         testOutput.Add("Unable to read error file: " + errorFile)%3B
-                        testOutput.Add(ex.Message)%3B
+                        testOutput.Add(ex.ToString())%3B
                     }
 
                     testOutput.Add(string.Empty)%3B
@@ -319,7 +319,7 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\","").
                     catch(Exception ex)
                     {
                         testOutput.Add("Unable to read output file: " + outputFile)%3B
-                        testOutput.Add(ex.Message)%3B
+                        testOutput.Add(ex.ToString())%3B
                     }
 
                     testOutput.Add("To run the test:")%3B