Shutdown build servers via named pipes

Directory.Packages.props

@@ -62,6 +62,7 @@
     <PackageVersion Include="Microsoft.FSharp.Compiler" Version="$(MicrosoftFSharpCompilerPackageVersion)" />
     <PackageVersion Include="Microsoft.Net.Compilers.Toolset.Framework" Version="$(MicrosoftNetCompilersToolsetFrameworkPackageVersion)" />
     <PackageVersion Include="Microsoft.Management.Infrastructure" Version="3.0.0" />
+    <PackageVersion Include="Microsoft.Net.BuildServerUtils" Version="$(MicrosoftNetBuildServerUtilsVersion)" />
     <PackageVersion Include="Microsoft.NET.HostModel" Version="$(MicrosoftNETHostModelVersion)" />
     <PackageVersion Include="Microsoft.NET.Sdk.Razor.SourceGenerators.Transport" Version="$(MicrosoftNETSdkRazorSourceGeneratorsTransportPackageVersion)" />
     <PackageVersion Include="Microsoft.NETCore.App.Runtime.win-x86" Version="$(MicrosoftNETCoreAppRuntimePackageVersion)" />

NuGet.config

@@ -32,6 +32,7 @@
     <add key="richnav" value="https://pkgs.dev.azure.com/azure-public/vside/_packaging/vs-buildservices/nuget/v3/index.json" />
     <!-- mstest dependencies -->
     <add key="test-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/test-tools/nuget/v3/index.json" />
+    <add key="general-testing" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json" />
   </packageSources>
   <disabledPackageSources>
     <clear />

eng/Version.Details.props

@@ -36,16 +36,16 @@ This file should be imported by eng/Versions.props
     <MicrosoftBuildLocalizationPackageVersion>17.15.0-preview-25416-109</MicrosoftBuildLocalizationPackageVersion>
     <MicrosoftBuildNuGetSdkResolverPackageVersion>7.0.0-preview.1.41709</MicrosoftBuildNuGetSdkResolverPackageVersion>
     <MicrosoftBuildTasksGitPackageVersion>10.0.0-beta.25416.109</MicrosoftBuildTasksGitPackageVersion>
-    <MicrosoftCodeAnalysisPackageVersion>5.0.0-2.25416.109</MicrosoftCodeAnalysisPackageVersion>
-    <MicrosoftCodeAnalysisBuildClientPackageVersion>5.0.0-2.25416.109</MicrosoftCodeAnalysisBuildClientPackageVersion>
-    <MicrosoftCodeAnalysisCSharpPackageVersion>5.0.0-2.25416.109</MicrosoftCodeAnalysisCSharpPackageVersion>
-    <MicrosoftCodeAnalysisCSharpCodeStylePackageVersion>5.0.0-2.25416.109</MicrosoftCodeAnalysisCSharpCodeStylePackageVersion>
-    <MicrosoftCodeAnalysisCSharpFeaturesPackageVersion>5.0.0-2.25416.109</MicrosoftCodeAnalysisCSharpFeaturesPackageVersion>
-    <MicrosoftCodeAnalysisCSharpWorkspacesPackageVersion>5.0.0-2.25416.109</MicrosoftCodeAnalysisCSharpWorkspacesPackageVersion>
-    <MicrosoftCodeAnalysisPublicApiAnalyzersPackageVersion>5.0.0-2.25416.109</MicrosoftCodeAnalysisPublicApiAnalyzersPackageVersion>
+    <MicrosoftCodeAnalysisPackageVersion>5.0.0-2.25421.2</MicrosoftCodeAnalysisPackageVersion>
+    <MicrosoftCodeAnalysisBuildClientPackageVersion>5.0.0-2.25421.2</MicrosoftCodeAnalysisBuildClientPackageVersion>
+    <MicrosoftCodeAnalysisCSharpPackageVersion>5.0.0-2.25421.2</MicrosoftCodeAnalysisCSharpPackageVersion>
+    <MicrosoftCodeAnalysisCSharpCodeStylePackageVersion>5.0.0-2.25421.2</MicrosoftCodeAnalysisCSharpCodeStylePackageVersion>
+    <MicrosoftCodeAnalysisCSharpFeaturesPackageVersion>5.0.0-2.25421.2</MicrosoftCodeAnalysisCSharpFeaturesPackageVersion>
+    <MicrosoftCodeAnalysisCSharpWorkspacesPackageVersion>5.0.0-2.25421.2</MicrosoftCodeAnalysisCSharpWorkspacesPackageVersion>
+    <MicrosoftCodeAnalysisPublicApiAnalyzersPackageVersion>5.0.0-2.25421.2</MicrosoftCodeAnalysisPublicApiAnalyzersPackageVersion>
     <MicrosoftCodeAnalysisRazorToolingInternalPackageVersion>10.0.0-preview.25416.109</MicrosoftCodeAnalysisRazorToolingInternalPackageVersion>
-    <MicrosoftCodeAnalysisWorkspacesCommonPackageVersion>5.0.0-2.25416.109</MicrosoftCodeAnalysisWorkspacesCommonPackageVersion>
-    <MicrosoftCodeAnalysisWorkspacesMSBuildPackageVersion>5.0.0-2.25416.109</MicrosoftCodeAnalysisWorkspacesMSBuildPackageVersion>
+    <MicrosoftCodeAnalysisWorkspacesCommonPackageVersion>5.0.0-2.25421.2</MicrosoftCodeAnalysisWorkspacesCommonPackageVersion>
+    <MicrosoftCodeAnalysisWorkspacesMSBuildPackageVersion>5.0.0-2.25421.2</MicrosoftCodeAnalysisWorkspacesMSBuildPackageVersion>
     <MicrosoftDeploymentDotNetReleasesPackageVersion>2.0.0-preview.1.25416.109</MicrosoftDeploymentDotNetReleasesPackageVersion>
     <MicrosoftDiaSymReaderPackageVersion>2.2.0-beta.25416.109</MicrosoftDiaSymReaderPackageVersion>
     <MicrosoftDotNetArcadeSdkPackageVersion>10.0.0-beta.25416.109</MicrosoftDotNetArcadeSdkPackageVersion>
@@ -69,8 +69,9 @@ This file should be imported by eng/Versions.props
     <MicrosoftExtensionsObjectPoolPackageVersion>10.0.0-rc.2.25416.109</MicrosoftExtensionsObjectPoolPackageVersion>
     <MicrosoftFSharpCompilerPackageVersion>14.0.100-rc2.25416.109</MicrosoftFSharpCompilerPackageVersion>
     <MicrosoftJSInteropPackageVersion>10.0.0-rc.2.25416.109</MicrosoftJSInteropPackageVersion>
-    <MicrosoftNetCompilersToolsetPackageVersion>5.0.0-2.25416.109</MicrosoftNetCompilersToolsetPackageVersion>
-    <MicrosoftNetCompilersToolsetFrameworkPackageVersion>5.0.0-2.25416.109</MicrosoftNetCompilersToolsetFrameworkPackageVersion>
+    <MicrosoftNetBuildServerUtilsPackageVersion>5.0.0-2.25421.2</MicrosoftNetBuildServerUtilsPackageVersion>
+    <MicrosoftNetCompilersToolsetPackageVersion>5.0.0-2.25421.2</MicrosoftNetCompilersToolsetPackageVersion>
+    <MicrosoftNetCompilersToolsetFrameworkPackageVersion>5.0.0-2.25421.2</MicrosoftNetCompilersToolsetFrameworkPackageVersion>
     <MicrosoftNETHostModelPackageVersion>10.0.0-rc.2.25416.109</MicrosoftNETHostModelPackageVersion>
     <MicrosoftNETILLinkTasksPackageVersion>10.0.0-rc.2.25416.109</MicrosoftNETILLinkTasksPackageVersion>
     <MicrosoftNETRuntimeEmscripten3156Cachewinx64PackageVersion>10.0.0-rc.2.25416.109</MicrosoftNETRuntimeEmscripten3156Cachewinx64PackageVersion>
@@ -209,6 +210,7 @@ This file should be imported by eng/Versions.props
     <MicrosoftExtensionsObjectPoolVersion>$(MicrosoftExtensionsObjectPoolPackageVersion)</MicrosoftExtensionsObjectPoolVersion>
     <MicrosoftFSharpCompilerVersion>$(MicrosoftFSharpCompilerPackageVersion)</MicrosoftFSharpCompilerVersion>
     <MicrosoftJSInteropVersion>$(MicrosoftJSInteropPackageVersion)</MicrosoftJSInteropVersion>
+    <MicrosoftNetBuildServerUtilsVersion>$(MicrosoftNetBuildServerUtilsPackageVersion)</MicrosoftNetBuildServerUtilsVersion>
     <MicrosoftNetCompilersToolsetVersion>$(MicrosoftNetCompilersToolsetPackageVersion)</MicrosoftNetCompilersToolsetVersion>
     <MicrosoftNetCompilersToolsetFrameworkVersion>$(MicrosoftNetCompilersToolsetFrameworkPackageVersion)</MicrosoftNetCompilersToolsetFrameworkVersion>
     <MicrosoftNETHostModelVersion>$(MicrosoftNETHostModelPackageVersion)</MicrosoftNETHostModelVersion>

src/Cli/Microsoft.DotNet.Cli.Utils/MSBuildForwardingAppWithoutLogging.cs

@@ -5,7 +5,7 @@
 
 using System.Diagnostics;
 using Microsoft.DotNet.Cli.Utils.Extensions;
-using Microsoft.DotNet.Cli;
+using Microsoft.Net.BuildServerUtils;
 
 namespace Microsoft.DotNet.Cli.Utils;
 
@@ -70,6 +70,8 @@ public MSBuildForwardingAppWithoutLogging(MSBuildArgs msbuildArgs, string? msbui
 
         EnvironmentVariable("MSBUILDUSESERVER", UseMSBuildServer ? "1" : "0");
 
+        EnvironmentVariable(BuildServerUtility.DotNetHostServerPath, GetHostServerPath(createDirectory: true));
+
         // If DOTNET_CLI_RUN_MSBUILD_OUTOFPROC is set or we're asked to execute a non-default binary, call MSBuild out-of-proc.
         if (AlwaysExecuteMSBuildOutOfProc || !string.Equals(MSBuildPath, defaultMSBuildPath, StringComparison.OrdinalIgnoreCase))
         {
@@ -135,6 +137,49 @@ public void EnvironmentVariable(string name, string? value)
         }
     }
 
+    public static string GetHostServerPath(bool createDirectory)
+    {
+        // If the path is set from outside, reuse it.
+        var hostServerPath = Env.GetEnvironmentVariable(BuildServerUtility.DotNetHostServerPath);
+        if (string.IsNullOrWhiteSpace(hostServerPath))
+        {
+            // Otherwise, construct a directory path under temp.
+
+            // If we're on a Unix machine then named pipes are implemented using Unix Domain Sockets.
+            // Most Unix systems have a maximum path length limit for Unix Domain Sockets, with Mac having a particularly short one.
+            // Mac also has a generated temp directory that can be quite long, leaving very little room for the actual pipe name.
+            // Fortunately, '/tmp' is mandated by POSIX to always be a valid temp directory, so we can use that instead.
+            const string dotnetServer = "dotnet-server";
+            string baseDirectory = OperatingSystem.IsWindows()
+                ? Path.Join(dotnetServer, Environment.UserName) // it's not a real path on Windows, just a name
+                : Path.Join("/tmp/", dotnetServer, Environment.UserName);
+
+            string sdkPathHashed = Sha256Hasher.HashWithNormalizedCasing(AppContext.BaseDirectory);
+
+            hostServerPath = Path.Join(baseDirectory, Product.TargetFrameworkVersion, sdkPathHashed);
+
+            const int limit = 104;
+            Debug.Assert(hostServerPath.Length < limit,
+                $"Path '{hostServerPath}' has length {hostServerPath.Length}. The limit is {limit} on Mac.");
+        }
+
+        // Create the directory on Linux (it's not a real directory on Windows, it's a virtual \\.\pipe\ namespace there).
+        if (createDirectory && !OperatingSystem.IsWindows())
+        {
+            try
+            {
+                PathUtility.CreateUserRestrictedDirectory(hostServerPath);
+            }
+            catch (Exception ex)
+            {
+                string details = CommandLoggingContext.IsVerbose ? ex.ToString() : ex.Message;
+                Reporter.Error.WriteLine($"Cannot create {BuildServerUtility.DotNetHostServerPath} '{hostServerPath}': {details}");
+            }
+        }
+
+        return hostServerPath;
+    }
+
     public int Execute()
     {
         if (_forwardingApp != null)

src/Cli/Microsoft.DotNet.Cli.Utils/Microsoft.DotNet.Cli.Utils.csproj

@@ -53,6 +53,7 @@
     <PackageReference Include="NuGet.ProjectModel" />
     <PackageReference Include="Microsoft.Build" ExcludeAssets="runtime" PrivateAssets="all" />
     <PackageReference Include="Microsoft.Build.Utilities.Core" ExcludeAssets="runtime" PrivateAssets="all" />
+    <PackageReference Include="Microsoft.Net.BuildServerUtils" />
     <PackageReference Include="System.CommandLine" />
     <PackageReference Include="System.IO.Hashing" />
   </ItemGroup>

src/Cli/Microsoft.DotNet.Cli.Utils/PathUtility.cs

@@ -89,6 +89,29 @@ public static void EnsureDirectoryExists(string? directoryPath)
         }
     }
 
+    public static string GetUserRestrictedTempDirectory()
+    {
+        // We want a location where permissions are expected to be restricted to the current user.
+        return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
+            ? Path.GetTempPath()
+            : Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
+    }
+
+#if NET
+    public static void CreateUserRestrictedDirectory(string path)
+    {
+        if (OperatingSystem.IsWindows())
+        {
+            Directory.CreateDirectory(path);
+        }
+        else
+        {
+            // NOTE: This modifies the permissions if needed, and throws if not possible.
+            Directory.CreateDirectory(path, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute);
+        }
+    }
+#endif
+
     public static bool TryDeleteDirectory(string directoryPath)
     {
         try

src/Cli/dotnet/BuildServer/BuildServerProvider.cs

@@ -22,6 +22,11 @@ internal class BuildServerProvider(
 
     public IEnumerable<IBuildServer> EnumerateBuildServers(ServerEnumerationFlags flags = ServerEnumerationFlags.All)
     {
+        if ((flags & ServerEnumerationFlags.Unified) == ServerEnumerationFlags.Unified)
+        {
+            yield return new UnifiedBuildServer();
+        }
+
         if ((flags & ServerEnumerationFlags.MSBuild) == ServerEnumerationFlags.MSBuild)
         {
             // Yield a single MSBuild server (handles server discovery itself)

src/Cli/dotnet/BuildServer/IBuildServer.cs

@@ -11,5 +11,5 @@ internal interface IBuildServer
 
     string Name { get; }
 
-    void Shutdown();
+    Task ShutdownAsync();
 }

src/Cli/dotnet/BuildServer/IBuildServerProvider.cs

@@ -9,10 +9,11 @@ namespace Microsoft.DotNet.Cli.BuildServer;
 internal enum ServerEnumerationFlags
 {
     None = 0,
-    MSBuild = 1,
-    VBCSCompiler = 2,
-    Razor = 4,
-    All = MSBuild | VBCSCompiler | Razor
+    MSBuild = 1 << 0,
+    VBCSCompiler = 1 << 1,
+    Razor = 1 << 2,
+    Unified = 1 << 3,
+    All = MSBuild | VBCSCompiler | Razor | Unified
 }
 
 internal interface IBuildServerProvider

src/Cli/dotnet/BuildServer/MSBuildServer.cs

@@ -13,8 +13,9 @@ internal class MSBuildServer : IBuildServer
 
     public string Name => CliStrings.MSBuildServer;
 
-    public void Shutdown()
+    public Task ShutdownAsync()
     {
         BuildManager.DefaultBuildManager.ShutdownAllNodes();
+        return Task.CompletedTask;
     }
 }

src/Cli/dotnet/BuildServer/RazorServer.cs

@@ -22,14 +22,15 @@ internal class RazorServer(
 
     public RazorPidFile PidFile { get; } = pidFile ?? throw new ArgumentNullException(nameof(pidFile));
 
-    public void Shutdown()
+    public Task ShutdownAsync()
     {
         if (!_fileSystem.File.Exists(PidFile.ServerPath.Value))
         {
             // The razor server path doesn't exist anymore so trying to shut it down would fail
             // Ensure the pid file is cleaned up so we don't try to shut it down again
             DeletePidFile();
-            return;
+
+            return Task.CompletedTask;
         }
 
         var command = _commandFactory
@@ -57,6 +58,8 @@ public void Shutdown()
         // After a successful shutdown, ensure the pid file is deleted
         // If the pid file was left behind due to a rude exit, this ensures we don't try to shut it down again
         DeletePidFile();
+
+        return Task.CompletedTask;
     }
 
     void DeletePidFile()

src/Cli/dotnet/BuildServer/UnifiedBuildServer.cs

@@ -0,0 +1,77 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using Microsoft.DotNet.Cli.Commands;
+using Microsoft.DotNet.Cli.Utils;
+using Microsoft.Net.BuildServerUtils;
+
+namespace Microsoft.DotNet.Cli.BuildServer;
+
+internal sealed class UnifiedBuildServer : IBuildServer
+{
+    public int ProcessId => 0; // Not used
+
+    public string Name => CliCommandStrings.UnifiedBuildServer;
+
+    public Task ShutdownAsync()
+    {
+        var hostServerPath = MSBuildForwardingAppWithoutLogging.GetHostServerPath(createDirectory: false);
+        var pipeFolder = BuildServerUtility.GetPipeFolder(hostServerPath);
+        Debug.Assert(pipeFolder != null);
+        Reporter.Output.WriteLine(CliCommandStrings.ShuttingDownUnifiedBuildServers, AppContext.BaseDirectory, pipeFolder);
+
+        return Task.WhenAll(EnumeratePipes(pipeFolder).Select(async file =>
+        {
+            try
+            {
+                if (!BuildServerUtility.TryParsePipePath(file, out int pid, out ReadOnlySpan<char> label))
+                {
+                    throw new GracefulException(CliCommandStrings.NamedPipeFileBadFormat, file);
+                }
+
+                Reporter.Output.WriteLine(CliCommandStrings.ShuttingDownServerWithPid, label.ToString(), pid);
+
+                // Connect to each pipe.
+                var client = BuildServerUtility.CreateClient(file);
+                await using var _ = client.ConfigureAwait(false);
+                await client.ConnectAsync().ConfigureAwait(false);
+
+                // Send any data to request shutdown.
+                byte[] data = [1];
+                await client.WriteAsync(data).ConfigureAwait(false);
+
+                // Wait for the process to exit.
+                using var process = Process.GetProcessById(pid);
+                await process.WaitForExitAsync().ConfigureAwait(false);
+            }
+            catch (Exception ex) when (ex is not GracefulException)
+            {
+                throw new GracefulException(string.Format(CliCommandStrings.NamedPipeShutdownError, file, ex.Message), ex);
+            }
+        }));
+
+        static IEnumerable<string> EnumeratePipes(string pipeFolder)
+        {
+            // On Windows, we need to enumerate all pipes and then filter them.
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                Debug.Assert(pipeFolder.EndsWith('\\'));
+                return Directory.EnumerateFiles(BuildServerUtility.WindowsPipePrefix)
+                    .Where(path => path.StartsWith(pipeFolder, StringComparison.OrdinalIgnoreCase) &&
+                        !path.AsSpan(pipeFolder.Length).ContainsAny('/', '\\'));
+            }
+
+            // On Unix, we can directly enumerate the files in the pipe folder.
+            try
+            {
+                return Directory.EnumerateFiles(pipeFolder);
+            }
+            catch (DirectoryNotFoundException)
+            {
+                Reporter.Output.WriteLine(CliCommandStrings.NamedPipeFolderNotFound, pipeFolder);
+                return [];
+            }
+        }
+    }
+}

src/Cli/dotnet/BuildServer/VBCSCompilerServer.cs

@@ -10,7 +10,7 @@
 
 namespace Microsoft.DotNet.Cli.BuildServer;
 
-internal class VBCSCompilerServer(ICommandFactory commandFactory = null) : IBuildServer
+internal class VBCSCompilerServer : IBuildServer
 {
     private static readonly string s_toolsetPackageName = "microsoft.net.sdk.compilers.toolset";
     private static readonly string s_vbcsCompilerExeFileName = "VBCSCompiler.exe";
@@ -22,19 +22,14 @@ internal class VBCSCompilerServer(ICommandFactory commandFactory = null) : IBuil
             "bincore",
             "VBCSCompiler.dll");
 
-    private readonly ICommandFactory _commandFactory = commandFactory ?? new DotNetCommandFactory(alwaysRunOutOfProc: true);
-
     public int ProcessId => 0; // Not yet used
 
     public string Name => CliStrings.VBCSCompilerServer;
 
-    public void Shutdown()
+    public Task ShutdownAsync()
     {
         List<string> errors = null;
 
-        // Shutdown the compiler from the SDK.
-        execute(_commandFactory.Create("exec", [VBCSCompilerPath, s_shutdownArg]), ref errors);
-
         // Shutdown toolset compilers.
         Reporter.Verbose.WriteLine($"Shutting down '{s_toolsetPackageName}' compilers.");
         var nuGetPackageRoot = SettingsUtility.GetGlobalPackagesFolder(Settings.LoadDefaultSettings(root: null));
@@ -61,6 +56,8 @@ public void Shutdown()
                     string.Join(Environment.NewLine, errors)));
         }
 
+        return Task.CompletedTask;
+
         static void execute(ICommand command, ref List<string> errors)
         {
             command = command