Add VSTestConsoleLocator and expand AppDomainTests with vstest.consol…

…e.exe tests

Co-authored-by: Youssef1313 <[email protected]>

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/AppDomainTests.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using Microsoft.MSTestV2.CLIAutomation;
 using Microsoft.Testing.Platform.Acceptance.IntegrationTests;
 using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers;
 
@@ -123,5 +124,113 @@ public async Task DiscoverTests_With_VSTest(bool? disableAppDomain)
         Assert.AreEqual(0, compilationResult.ExitCode);
     }
 
+    [TestMethod]
+    [DataRow(true)]
+    [DataRow(false)]
+    [DataRow(null)]
+    public async Task RunTests_With_VSTestConsole_Directly(bool? disableAppDomain)
+    {
+        using TestAsset testAsset = await TestAsset.GenerateAssetAsync(
+            AssetName,
+            SingleTestSourceCode
+            .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)
+            .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetFramework[0]));
+
+        // Build the test project
+        DotnetMuxerResult buildResult = await DotnetCli.RunAsync(
+            $"build {testAsset.TargetAssetPath} -c Debug",
+            AcceptanceFixture.NuGetGlobalPackagesFolder.Path,
+            workingDirectory: testAsset.TargetAssetPath,
+            cancellationToken: TestContext.CancellationToken);
+        Assert.AreEqual(0, buildResult.ExitCode, $"Build failed: {buildResult.StandardOutput}");
+
+        // Get the DLL path
+        string dllPath = Path.Combine(testAsset.TargetAssetPath, "bin", "Debug", TargetFrameworks.NetFramework[0], $"{AssetName}.dll");
+        Assert.IsTrue(File.Exists(dllPath), $"Test DLL not found at {dllPath}");
+
+        // Prepare run settings
+        string runSettings = disableAppDomain switch
+        {
+            true => "<RunSettings><RunConfiguration><DisableAppDomain>true</DisableAppDomain></RunConfiguration></RunSettings>",
+            false => "<RunSettings><RunConfiguration><DisableAppDomain>false</DisableAppDomain></RunConfiguration></RunSettings>",
+            null => string.Empty,
+        };
+
+        // Run tests using vstest.console.exe directly
+        string vstestConsolePath = VSTestConsoleLocator.GetConsoleRunnerPath();
+        string arguments = $"\"{vstestConsolePath}\" \"{dllPath}\"";
+        if (!string.IsNullOrEmpty(runSettings))
+        {
+            string runSettingsPath = Path.Combine(testAsset.TargetAssetPath, "test.runsettings");
+            await File.WriteAllTextAsync(runSettingsPath, runSettings);
+            arguments += $" /Settings:\"{runSettingsPath}\"";
+        }
+
+        using var commandLine = new CommandLine();
+        int exitCode = await commandLine.RunAsyncAndReturnExitCodeAsync(
+            arguments,
+            workingDirectory: testAsset.TargetAssetPath,
+            cancellationToken: TestContext.CancellationToken);
+
+        Assert.AreEqual(0, exitCode, $"Tests failed: {commandLine.StandardOutput}");
+        Assert.IsTrue(commandLine.StandardOutput.Contains("Passed   2") || commandLine.StandardOutput.Contains("Passed: 2"), 
+            $"Expected 2 passed tests but got: {commandLine.StandardOutput}");
+    }
+
+    [TestMethod]
+    [DataRow(true)]
+    [DataRow(false)]
+    [DataRow(null)]
+    public async Task DiscoverTests_With_VSTestConsole_Directly(bool? disableAppDomain)
+    {
+        using TestAsset testAsset = await TestAsset.GenerateAssetAsync(
+            AssetName,
+            SingleTestSourceCode
+            .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)
+            .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetFramework[0]));
+
+        // Build the test project
+        DotnetMuxerResult buildResult = await DotnetCli.RunAsync(
+            $"build {testAsset.TargetAssetPath} -c Debug",
+            AcceptanceFixture.NuGetGlobalPackagesFolder.Path,
+            workingDirectory: testAsset.TargetAssetPath,
+            cancellationToken: TestContext.CancellationToken);
+        Assert.AreEqual(0, buildResult.ExitCode, $"Build failed: {buildResult.StandardOutput}");
+
+        // Get the DLL path
+        string dllPath = Path.Combine(testAsset.TargetAssetPath, "bin", "Debug", TargetFrameworks.NetFramework[0], $"{AssetName}.dll");
+        Assert.IsTrue(File.Exists(dllPath), $"Test DLL not found at {dllPath}");
+
+        // Prepare run settings
+        string runSettings = disableAppDomain switch
+        {
+            true => "<RunSettings><RunConfiguration><DisableAppDomain>true</DisableAppDomain></RunConfiguration></RunSettings>",
+            false => "<RunSettings><RunConfiguration><DisableAppDomain>false</DisableAppDomain></RunConfiguration></RunSettings>",
+            null => string.Empty,
+        };
+
+        // Run discovery using vstest.console.exe directly
+        string vstestConsolePath = VSTestConsoleLocator.GetConsoleRunnerPath();
+        string arguments = $"\"{vstestConsolePath}\" \"{dllPath}\" /ListTests";
+        if (!string.IsNullOrEmpty(runSettings))
+        {
+            string runSettingsPath = Path.Combine(testAsset.TargetAssetPath, "test.runsettings");
+            await File.WriteAllTextAsync(runSettingsPath, runSettings);
+            arguments += $" /Settings:\"{runSettingsPath}\"";
+        }
+
+        using var commandLine = new CommandLine();
+        int exitCode = await commandLine.RunAsyncAndReturnExitCodeAsync(
+            arguments,
+            workingDirectory: testAsset.TargetAssetPath,
+            cancellationToken: TestContext.CancellationToken);
+
+        Assert.AreEqual(0, exitCode, $"Discovery failed: {commandLine.StandardOutput}");
+        Assert.IsTrue(commandLine.StandardOutput.Contains("AppDomainTests.UnitTest1.TestMethod1"), 
+            $"Expected to find TestMethod1 but got: {commandLine.StandardOutput}");
+        Assert.IsTrue(commandLine.StandardOutput.Contains("AppDomainTests.UnitTest1.TestMethod2"), 
+            $"Expected to find TestMethod2 but got: {commandLine.StandardOutput}");
+    }
+
     public TestContext TestContext { get; set; }
 }

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSTest.Acceptance.IntegrationTests.csproj

@@ -13,11 +13,13 @@
     <Compile Include="$(RepoRoot)test\IntegrationTests\Microsoft.Testing.Platform.Acceptance.IntegrationTests\Helpers\AcceptanceFixture.cs" Link="Helpers\AcceptanceFixture.cs" />
     <Compile Include="$(RepoRoot)test\IntegrationTests\Microsoft.Testing.Platform.Acceptance.IntegrationTests\Helpers\AcceptanceTestBase.cs" Link="Helpers\AcceptanceTestBase.cs" />
     <Compile Include="$(RepoRoot)test\IntegrationTests\Microsoft.Testing.Platform.Acceptance.IntegrationTests\ServerMode\**\*.cs" Link="ServerMode\%(RecursiveDir)%(FileName)%(Extension)" />
+    <Compile Include="$(RepoRoot)test\Utilities\Automation.CLI\VSTestConsoleLocator.cs" Link="Helpers\VSTestConsoleLocator.cs" />
   </ItemGroup>
 
   <ItemGroup>
     <PackageReference Include="MSBuild.StructuredLogger" />
     <PackageReference Include="StreamJsonRpc" />
+    <PackageReference Include="Microsoft.TestPlatform" />
   </ItemGroup>
 
   <!-- Packages needed for the test assets but that we don't want to reference -->

test/Utilities/Automation.CLI/CLITestBase.e2e.cs

@@ -15,7 +15,7 @@ public abstract partial class CLITestBase
     protected CLITestBase()
     {
         s_vsTestConsoleWrapper = new(
-            GetConsoleRunnerPath(),
+            VSTestConsoleLocator.GetConsoleRunnerPath(),
             new()
             {
                 EnvironmentVariables = new()
@@ -65,51 +65,7 @@ public void InvokeVsTestForExecution(string[] sources, string runSettings = "",
         }
     }
 
-    public static string GetNugetPackageFolder()
-    {
-        string nugetPackagesFolderPath = Environment.GetEnvironmentVariable("NUGET_PACKAGES");
-        if (!string.IsNullOrEmpty(nugetPackagesFolderPath))
-        {
-            Assert.IsTrue(Directory.Exists(nugetPackagesFolderPath), $"Found environment variable 'NUGET_PACKAGES' and NuGet package folder '{nugetPackagesFolderPath}' should exist");
-
-            return nugetPackagesFolderPath;
-        }
-
-        string userProfile = Environment.GetEnvironmentVariable("USERPROFILE");
-        nugetPackagesFolderPath = Path.Combine(userProfile, ".nuget", "packages");
-        Assert.IsTrue(Directory.Exists(nugetPackagesFolderPath), $"NuGet package folder '{nugetPackagesFolderPath}' should exist");
-
-        return nugetPackagesFolderPath;
-    }
 
-    /// <summary>
-    /// Gets the path to <c>vstest.console.exe</c>.
-    /// </summary>
-    /// <returns>Full path to <c>vstest.console.exe</c>.</returns>
-    public static string GetConsoleRunnerPath()
-    {
-        string testPlatformNuGetPackageFolder = Path.Combine(
-            GetNugetPackageFolder(),
-            TestPlatformCLIPackageName,
-            GetTestPlatformVersion());
-        if (!Directory.Exists(testPlatformNuGetPackageFolder))
-        {
-            throw new DirectoryNotFoundException($"Test platform NuGet package folder '{testPlatformNuGetPackageFolder}' does not exist");
-        }
-
-        string vstestConsolePath = Path.Combine(
-            testPlatformNuGetPackageFolder,
-            "tools",
-            "net462",
-            "Common7",
-            "IDE",
-            "Extensions",
-            "TestPlatform",
-            "vstest.console.exe");
-        return !File.Exists(vstestConsolePath)
-            ? throw new InvalidOperationException($"Could not find vstest.console.exe in {vstestConsolePath}")
-            : vstestConsolePath;
-    }
 
     /// <summary>
     /// Validate if the discovered tests list contains provided tests.

test/Utilities/Automation.CLI/VSTestConsoleLocator.cs

@@ -0,0 +1,85 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Assert = Microsoft.VisualStudio.TestTools.UnitTesting.Assert;
+
+namespace Microsoft.MSTestV2.CLIAutomation;
+
+/// <summary>
+/// Helper class to locate vstest.console.exe.
+/// </summary>
+public static class VSTestConsoleLocator
+{
+    private const string TestPlatformCLIPackageName = "Microsoft.TestPlatform";
+
+    /// <summary>
+    /// Gets the path to <c>vstest.console.exe</c>.
+    /// </summary>
+    /// <returns>Full path to <c>vstest.console.exe</c>.</returns>
+    public static string GetConsoleRunnerPath()
+    {
+        string testPlatformNuGetPackageFolder = Path.Combine(
+            GetNugetPackageFolder(),
+            TestPlatformCLIPackageName,
+            GetTestPlatformVersion());
+        if (!Directory.Exists(testPlatformNuGetPackageFolder))
+        {
+            throw new DirectoryNotFoundException($"Test platform NuGet package folder '{testPlatformNuGetPackageFolder}' does not exist");
+        }
+
+        string vstestConsolePath = Path.Combine(
+            testPlatformNuGetPackageFolder,
+            "tools",
+            "net462",
+            "Common7",
+            "IDE",
+            "Extensions",
+            "TestPlatform",
+            "vstest.console.exe");
+        return !File.Exists(vstestConsolePath)
+            ? throw new InvalidOperationException($"Could not find vstest.console.exe in {vstestConsolePath}")
+            : vstestConsolePath;
+    }
+
+    private static string GetNugetPackageFolder()
+    {
+        string nugetPackagesFolderPath = Environment.GetEnvironmentVariable("NUGET_PACKAGES");
+        if (!string.IsNullOrEmpty(nugetPackagesFolderPath))
+        {
+            Assert.IsTrue(Directory.Exists(nugetPackagesFolderPath), $"Found environment variable 'NUGET_PACKAGES' and NuGet package folder '{nugetPackagesFolderPath}' should exist");
+
+            return nugetPackagesFolderPath;
+        }
+
+        string userProfile = Environment.GetEnvironmentVariable("USERPROFILE");
+        nugetPackagesFolderPath = Path.Combine(userProfile, ".nuget", "packages");
+        Assert.IsTrue(Directory.Exists(nugetPackagesFolderPath), $"NuGet package folder '{nugetPackagesFolderPath}' should exist");
+
+        return nugetPackagesFolderPath;
+    }
+
+    private static string GetTestPlatformVersion()
+    {
+        string cpmFilePath = Path.Combine(GetArtifactsBinFolderPath(), "..", "..", "Directory.Packages.props");
+        using FileStream fileStream = File.OpenRead(cpmFilePath);
+#pragma warning disable CA3075 // Insecure DTD processing in XML
+        using var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false };
+#pragma warning restore CA3075 // Insecure DTD processing in XML
+        var cpmXml = new XmlDocument();
+        cpmXml.Load(xmlTextReader);
+
+        XmlNode testSdkVersion = cpmXml.DocumentElement.SelectSingleNode("PropertyGroup/MicrosoftNETTestSdkVersion");
+
+        return testSdkVersion.InnerText;
+    }
+
+    private static string GetArtifactsBinFolderPath()
+    {
+        string assemblyLocation = Assembly.GetExecutingAssembly().Location;
+
+        string artifactsBinFolder = Path.GetFullPath(Path.Combine(assemblyLocation, @"..\..\..\.."));
+        Assert.IsTrue(Directory.Exists(artifactsBinFolder));
+
+        return artifactsBinFolder;
+    }
+}