Add support for running tests marked with <RequiresProcessIsolation>t…

…rue</RequiresProcessIsolation> in a separate process using the cmd/sh build scripts.

src/tests/Common/CoreCLRTestLibrary/CoreCLRTestLibrary.csproj

@@ -16,6 +16,10 @@
     <Compile Include="HostPolicyMock.cs" />
     <Compile Include="XPlatformUtils.cs" />
     <Compile Include="CoreClrConfigurationDetection.cs" />
+    <Compile Include="OutOfProcessTest.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="..\Coreclr.TestWrapper\CoreclrTestWrapperLib.cs" Link="CoreclrTestWrapperLib.cs" />
   </ItemGroup>
 
   <ItemGroup>

src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs

@@ -0,0 +1,132 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using CoreclrTestLib;
+using Xunit;
+
+namespace TestLibrary
+{
+    public static class OutOfProcessTest
+    {
+        internal static bool runningInWindows;
+        internal static string reportBase;
+        internal static string testBinaryBase;
+        internal static string helixUploadRoot;
+
+        static OutOfProcessTest()
+        {
+            reportBase = Directory.GetCurrentDirectory();
+            testBinaryBase = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!;
+            helixUploadRoot = Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT");
+            if (!String.IsNullOrEmpty(helixUploadRoot))
+            {
+                reportBase = Path.Combine(Path.GetFullPath(helixUploadRoot), "Reports");
+            }
+
+            if (String.IsNullOrEmpty(reportBase))
+            {
+                reportBase = Path.Combine(testBinaryBase, "Reports");
+            }
+            else
+            {
+                reportBase = Path.GetFullPath(reportBase);
+            }
+        }
+
+        public static void RunOutOfProcessTest(string basePath, string assemblyPath)
+        {
+            int ret = -100;
+            string outputFile = System.IO.Path.GetFullPath(reportBase + assemblyPath + "output.txt");
+            string errorFile = System.IO.Path.GetFullPath(reportBase + assemblyPath + "error.txt");
+            string outputDir = System.IO.Path.GetDirectoryName(outputFile)!;
+            string testExecutable = null;
+            Exception infraEx = null;
+
+            try
+            {
+                CoreclrTestWrapperLib wrapper = new CoreclrTestWrapperLib();
+
+                if (OperatingSystem.IsWindows())
+                {
+                    testExecutable = Path.Combine(basePath, Path.ChangeExtension(assemblyPath, ".cmd"));
+                }
+                else
+                {
+                    testExecutable = Path.Combine(basePath, Path.ChangeExtension(assemblyPath.Replace("\\", "/"), ".sh"));
+                }
+
+                System.IO.Directory.CreateDirectory(reportBase + Path.GetDirectoryName(assemblyPath));
+
+                ret = wrapper.RunTest(testExecutable, outputFile, errorFile, Assembly.GetEntryAssembly()!.FullName!, testBinaryBase, outputDir);
+            }
+            catch (Exception ex)
+            {
+                infraEx = ex;
+            }
+
+            if (infraEx != null)
+            {
+                Assert.True(false, "Test Infrastructure Failure: " + infraEx.ToString());
+            }
+            else
+            {
+                List<string> testOutput = new List<string>();
+
+                try
+                {
+                    testOutput.AddRange(System.IO.File.ReadAllLines(errorFile));
+                }
+                catch (Exception ex)
+                {
+                    testOutput.Add("Unable to read error file: " + errorFile);
+                    testOutput.Add(ex.ToString());
+                }
+
+                testOutput.Add(string.Empty);
+                testOutput.Add("Return code:      " + ret);
+                testOutput.Add("Raw output file:      " + outputFile);
+                testOutput.Add("Raw output:");
+
+                try
+                {
+                    testOutput.AddRange(System.IO.File.ReadAllLines(outputFile));
+                }
+                catch (Exception ex)
+                {
+                    testOutput.Add("Unable to read output file: " + outputFile);
+                    testOutput.Add(ex.ToString());
+                }
+
+                testOutput.Add("To run the test:");
+                testOutput.Add("Set up CORE_ROOT and run.");
+                testOutput.Add("> " + testExecutable);
+
+                var unicodeControlCharsRegex = new Regex("%5C%5Cp{C}+");
+
+                // Remove all characters that have no visual or spatial representation.
+                for (int i = 0; i < testOutput.Count; i++)
+                {
+                    string line = testOutput[i];
+                    line = unicodeControlCharsRegex.Replace(line, string.Empty);
+                    testOutput[i] = line;
+                }
+
+                foreach (string line in testOutput)
+                {
+                    Console.WriteLine(line);
+                }
+
+                Assert.True(ret == CoreclrTestWrapperLib.EXIT_SUCCESS_CODE, string.Join(Environment.NewLine, testOutput));
+            }
+        }
+    }
+}

src/tests/Common/Coreclr.TestWrapper/CoreclrTestWrapperLib.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 //
+#nullable disable
 
 using System;
 using System.Collections.Generic;
@@ -196,7 +197,7 @@ public class CoreclrTestWrapperLib
     {
         public const int EXIT_SUCCESS_CODE = 0;
         public const string TIMEOUT_ENVIRONMENT_VAR = "__TestTimeout";
-        
+
         // Default timeout set to 10 minutes
         public const int DEFAULT_TIMEOUT_MS = 1000 * 60 * 10;
 
@@ -289,7 +290,7 @@ public int RunTest(string executable, string outputFile, string errorFile, strin
             Debug.Assert(outputFile != errorFile);
 
             int exitCode = -100;
-            
+
             // If a timeout was given to us by an environment variable, use it instead of the default
             // timeout.
             string environmentVar = Environment.GetEnvironmentVariable(TIMEOUT_ENVIRONMENT_VAR);
@@ -387,7 +388,5 @@ public int RunTest(string executable, string outputFile, string errorFile, strin
 
             return exitCode;
         }
-
-
     }
 }

src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.IO;
 using System.Text;
 using Microsoft.CodeAnalysis;
 
@@ -160,7 +161,7 @@ private static string GetPlatformConditionFromTestPlatform(Xunit.TestPlatforms p
         }
         if (platform.HasFlag(Xunit.TestPlatforms.tvOS))
         {
-            platformCheckConditions.Add("global::System.OperatingSystem.IsTVOS()");
+            platformCheckConditions.Add("global::System.OperatingSystem.IsTvOS()");
         }
         if (platform.HasFlag(Xunit.TestPlatforms.MacCatalyst))
         {
@@ -228,14 +229,34 @@ public override bool Equals(object obj)
     }
 }
 
-class NoTestReporting : ITestReporterWrapper
+sealed class OutOfProcessTest : ITestInfo
+{
+    public OutOfProcessTest(string displayName, string relativeAssemblyPath)
+    {
+        Method = displayName;
+        TestNameExpression = $"@\"{displayName}\"";
+        ExecutionStatement = $@"TestLibrary.OutOfProcessTest.RunOutOfProcessTest(typeof(Program).Assembly.Location, @""{relativeAssemblyPath}"");";
+    }
+
+    public string TestNameExpression { get; }
+
+    public string Method { get; }
+
+    public string ContainingType => "OutOfProcessTest";
+
+    private string ExecutionStatement { get; }
+
+    public string GenerateTestExecution(ITestReporterWrapper testReporterWrapper) => testReporterWrapper.WrapTestExecutionWithReporting(ExecutionStatement, this);
+}
+
+sealed class NoTestReporting : ITestReporterWrapper
 {
     public string WrapTestExecutionWithReporting(string testExecution, ITestInfo test) => testExecution;
 
     public string GenerateSkippedTestReporting(ITestInfo skippedTest) => string.Empty;
 }
 
-class WrapperLibraryTestSummaryReporting : ITestReporterWrapper
+sealed class WrapperLibraryTestSummaryReporting : ITestReporterWrapper
 {
     private string _summaryLocalIdentifier;
 
@@ -252,10 +273,10 @@ public string WrapTestExecutionWithReporting(string testExecutionExpression, ITe
         builder.AppendLine($"TimeSpan testStart = stopwatch.Elapsed;");
         builder.AppendLine("try {");
         builder.AppendLine(testExecutionExpression);
-        builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", \"{test.Method}\", stopwatch.Elapsed - testStart);");
+        builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart);");
         builder.AppendLine("}");
         builder.AppendLine("catch (System.Exception ex) {");
-        builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", \"{test.Method}\", stopwatch.Elapsed - testStart, ex);");
+        builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, ex);");
         builder.AppendLine("}");
 
         builder.AppendLine("}");

src/tests/Common/XUnitWrapperGenerator/OptionsHelper.cs

@@ -6,10 +6,13 @@ namespace XUnitWrapperGenerator;
 
 public static class OptionsHelper
 {
-    public const string ReferenceSystemPrivateCoreLibOption = "build_property.ReferenceSystemPrivateCoreLib";
-    public const string IsMergedTestRunnerAssemblyOption = "build_property.IsMergedTestRunnerAssembly";
-    public const string PriorityOption = "build_property.Priority";
-    public const string RuntimeFlavorOption = "build_property.RuntimeFlavor";
+    private const string ReferenceSystemPrivateCoreLibOption = "build_property.ReferenceSystemPrivateCoreLib";
+    private const string IsMergedTestRunnerAssemblyOption = "build_property.IsMergedTestRunnerAssembly";
+    private const string PriorityOption = "build_property.Priority";
+    private const string RuntimeFlavorOption = "build_property.RuntimeFlavor";
+    private const string IsOutOfProcessTestAssemblyOption = "build_metadata.AdditionalFiles.IsOutOfProcessTestAssembly";
+    private const string TestAssemblyRelativePathOption = "build_metadata.AdditionalFiles.TestAssemblyRelativePath";
+    private const string TestDisplayNameOption = "build_metadata.AdditionalFiles.TestDisplayName";
 
     private static bool GetBoolOption(this AnalyzerConfigOptions options, string key)
     {
@@ -32,4 +35,10 @@ private static bool GetBoolOption(this AnalyzerConfigOptions options, string key
     internal static int? Priority(this AnalyzerConfigOptions options) => options.GetIntOption(PriorityOption);
 
     internal static string RuntimeFlavor(this AnalyzerConfigOptions options) => options.TryGetValue(RuntimeFlavorOption, out string? flavor) ? flavor : "CoreCLR";
+
+    internal static bool IsOutOfProcessTestAssembly(this AnalyzerConfigOptions options) => options.GetBoolOption(IsOutOfProcessTestAssemblyOption);
+
+    internal static string? TestAssemblyRelativePath(this AnalyzerConfigOptions options) => options.TryGetValue(TestAssemblyRelativePathOption, out string? flavor) ? flavor : null;
+
+    internal static string? TestDisplayName(this AnalyzerConfigOptions options) => options.TryGetValue(TestDisplayNameOption, out string? flavor) ? flavor : null;
 }

src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs

@@ -32,7 +32,24 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
             return visitor.Visit(data.Right.GetAssemblyOrModuleSymbol(data.Left))!;
         });
 
-        var assemblyName = context.CompilationProvider.Select((comp, ct) => comp.Assembly.MetadataName);
+        var outOfProcessTests = context.AdditionalTextsProvider.Combine(context.AnalyzerConfigOptionsProvider).SelectMany((data, ct) =>
+        {
+            var (file, options) = data;
+
+            AnalyzerConfigOptions fileOptions = options.GetOptions(file);
+
+            if (fileOptions.IsOutOfProcessTestAssembly())
+            {
+                string? assemblyPath = fileOptions.TestAssemblyRelativePath();
+                string? testDisplayName = fileOptions.TestDisplayName();
+                if (assemblyPath is not null && testDisplayName is not null)
+                {
+                    return ImmutableArray.Create<ITestInfo>(new OutOfProcessTest(testDisplayName, assemblyPath));
+                }
+            }
+
+            return ImmutableArray<ITestInfo>.Empty;
+        });
 
         var allMethods = methodsInSource.Collect().Combine(methodsInReferencedAssemblies.Collect()).SelectMany((methods, ct) => methods.Left.AddRange(methods.Right));
 
@@ -48,12 +65,16 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
             return aliasMap.ToImmutable();
         }).WithComparer(new ImmutableDictionaryValueComparer<string, string>(EqualityComparer<string>.Default));
 
+        var assemblyName = context.CompilationProvider.Select((comp, ct) => comp.Assembly.MetadataName);
+
         context.RegisterImplementationSourceOutput(
             allMethods
             .Combine(context.AnalyzerConfigOptionsProvider)
             .Combine(aliasMap)
             .SelectMany((data, ct) => ImmutableArray.CreateRange(GetTestMethodInfosForMethod(data.Left.Left, data.Left.Right, data.Right)))
             .Collect()
+            .Combine(outOfProcessTests.Collect())
+            .Select((tests, ct) => tests.Left.AddRange(tests.Right))
             .Combine(aliasMap)
             .Combine(context.AnalyzerConfigOptionsProvider)
             .Combine(assemblyName),
@@ -318,6 +339,7 @@ private static IEnumerable<ITestInfo> GetTestMethodInfosForMethod(IMethodSymbol
                     if (options.GlobalOptions.RuntimeFlavor() != "CoreCLR")
                     {
                         // If we're building tests not for CoreCLR, we can skip handling the specifics of the SkipOnCoreClrAttribute.
+                        testInfos = ImmutableArray<ITestInfo>.Empty;
                         break;
                     }
 
@@ -508,7 +530,12 @@ private static ImmutableArray<ITestInfo> DecorateWithSkipOnPlatform(ImmutableArr
         options.GlobalOptions.TryGetValue("build_property.TargetOS", out string? targetOS);
         Xunit.TestPlatforms targetPlatform = GetPlatformForTargetOS(targetOS);
 
-        if (platformsToSkip.HasFlag(targetPlatform))
+        if (platformsToSkip == 0)
+        {
+            // In this case, we don't need to skip any platforms
+            return testInfos;
+        }
+        else if (platformsToSkip.HasFlag(targetPlatform))
         {
             // If the target platform is skipped, then we don't have any tests to emit.
             return ImmutableArray<ITestInfo>.Empty;

src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.props

@@ -8,5 +8,16 @@
     <!-- Properties that influence test harness generation -->
     <CompilerVisibleProperty Include="ReferenceSystemPrivateCoreLib" />
     <CompilerVisibleProperty Include="IsMergedTestRunnerAssembly" />
+    <CompilerVisibleItemMetadata Include="AdditionalFiles" MetadataName="IsOutOfProcessTestAssembly" />
+    <CompilerVisibleItemMetadata Include="AdditionalFiles" MetadataName="TestAssemblyRelativePath" />
+    <CompilerVisibleItemMetadata Include="AdditionalFiles" MetadataName="TestDisplayName" />
   </ItemGroup>
+
+  <Target Name="AddOutOfProcessTestAssembliesToCompilation" AfterTargets="ResolveProjectReferences">
+    <ItemGroup>
+      <OutOfProcessTests IsOutOfProcessTestAssembly="true" TestAssemblyRelativePath="$([MSBuild]::MakeRelative('$(OutputPath)', '%(Identity)'))" TestDisplayName="$([MSBuild]::MakeRelative('$(TestBinDir)', '%(Identity)'))" />
+      <!-- Change the extension to .cmd to match the existing test names for out-of-proc test runs. -->
+      <AdditionalFiles Include="@(OutOfProcessTests)" TestDisplayName="$([System.IO.Path]::ChangeExtension('%(TestDisplayName)', '.cmd'))" />
+    </ItemGroup>
+  </Target>
 </Project>

src/tests/Common/external/external.csproj

@@ -63,6 +63,7 @@
     <PackageToInclude Include="Microsoft.CodeAnalysis.CSharp"/>
     <PackageToInclude Include="Microsoft.CodeAnalysis.VisualBasic"/>
     <PackageToInclude Include="CommandLineParser"/>
+    <PackageToInclude Include="Microsoft.DotNet.XUnitExtensions" />
 
     <PackageReference Include="Microsoft.CodeAnalysis.Compilers" Version="1.1.1" />
     <PackageReference Include="CommandLineParser" Version="2.1.1-beta" />

src/tests/Directory.Build.targets

@@ -409,6 +409,23 @@
 
   </Target>
 
+  <Target Name="GetRequiresProcessIsolation" Returns="@(ProjectRequiringProcessIsolation)">
+    <ItemGroup Condition="'$(RequiresProcessIsolation)' == 'true'">
+      <ProjectRequiringProcessIsolation Include="$(MSBuildProjectFullPath)" />
+    </ItemGroup>
+  </Target>
+
+  <Target Name="DiscoverProcessIsolatedTestProjectReferences" BeforeTargets="AssignProjectConfiguration">
+    <MSBuild Projects="@(ProjectReference)" Targets="GetRequiresProcessIsolation" SkipNonexistentTargets="true">
+      <Output TaskParameter="TargetOutputs" ItemName="ReferencedProjectRequiringProcessIsolation" />
+    </MSBuild>
+
+    <ItemGroup>
+      <ProjectReference Remove="@(ReferencedProjectRequiringProcessIsolation)" />
+      <ProjectReference Include="@(ReferencedProjectRequiringProcessIsolation)" OutputItemType="OutOfProcessTests" BuildReference="true" ReferenceOutputAssembly="false" />
+    </ItemGroup>
+  </Target>
+
   <Import Project="$(RepoRoot)eng/liveBuilds.targets" />
   <Import Project="$(MSBuildProjectFullPath).targets" Condition="Exists('$(MSBuildProjectFullPath).targets')"/>