Fallback when server client fails due to mutex connection timeout error.

src/Build/BackEnd/Client/MSBuildClient.cs

@@ -8,7 +8,6 @@
 using System.Globalization;
 using System.IO;
 using System.IO.Pipes;
-using System.Linq;
 using System.Threading;
 using Microsoft.Build.BackEnd;
 using Microsoft.Build.BackEnd.Client;
@@ -165,33 +164,46 @@ public MSBuildClientExitResult Execute(CancellationToken cancellationToken)
 #endif
 
             CommunicationsUtilities.Trace("Executing build with command line '{0}'", descriptiveCommandLine);
-            bool serverIsAlreadyRunning = ServerIsRunning();
-            if (KnownTelemetry.BuildTelemetry != null)
-            {
-                KnownTelemetry.BuildTelemetry.InitialServerState = serverIsAlreadyRunning ? "hot" : "cold";
-            }
-            if (!serverIsAlreadyRunning)
+
+            try
             {
-                CommunicationsUtilities.Trace("Server was not running. Starting server now.");
-                if (!TryLaunchServer())
+                bool serverIsAlreadyRunning = ServerIsRunning();
+                if (KnownTelemetry.BuildTelemetry != null)
+                {
+                    KnownTelemetry.BuildTelemetry.InitialServerState = serverIsAlreadyRunning ? "hot" : "cold";
+                }
+                if (!serverIsAlreadyRunning)
                 {
-                    _exitResult.MSBuildClientExitType = MSBuildClientExitType.LaunchError;
+                    CommunicationsUtilities.Trace("Server was not running. Starting server now.");
+                    if (!TryLaunchServer())
+                    {
+                        _exitResult.MSBuildClientExitType = (_exitResult.MSBuildClientExitType == MSBuildClientExitType.Success) ? MSBuildClientExitType.LaunchError : _exitResult.MSBuildClientExitType;
+                        return _exitResult;
+                    }
+                }
+
+                // Check that server is not busy.
+                bool serverWasBusy = ServerWasBusy();
+                if (serverWasBusy)
+                {
+                    CommunicationsUtilities.Trace("Server is busy, falling back to former behavior.");
+                    _exitResult.MSBuildClientExitType = MSBuildClientExitType.ServerBusy;
                     return _exitResult;
                 }
-            }
 
-            // Check that server is not busy.
-            bool serverWasBusy = ServerWasBusy();
-            if (serverWasBusy)
-            {
-                CommunicationsUtilities.Trace("Server is busy, falling back to former behavior.");
-                _exitResult.MSBuildClientExitType = MSBuildClientExitType.ServerBusy;
-                return _exitResult;
+                // Connect to server.
+                if (!TryConnectToServer(serverIsAlreadyRunning ? 1_000 : 20_000))
+                {
+                    return _exitResult;
+                }
             }
-
-            // Connect to server.
-            if (!TryConnectToServer(serverIsAlreadyRunning ? 1_000 : 20_000))
+            catch (IOException ex) when (ex is not PathTooLongException)
             {
+                // For unknown root cause, Mutex.TryOpenExisting can sometimes throw 'Connection timed out' exception preventing to obtain the build server state through it (Running or not, Busy or not).
+                // See: https://github.com/dotnet/msbuild/issues/7993
+                CommunicationsUtilities.Trace("Failed to obtain the current build server state: {0}", ex);
+                CommunicationsUtilities.Trace("HResult: {0}.", ex.HResult);
+                _exitResult.MSBuildClientExitType = MSBuildClientExitType.UnknownServerState;
                 return _exitResult;
             }
 
@@ -465,23 +477,34 @@ private bool TrySendPacket(Func<INodePacket> packetResolver)
         private bool TryLaunchServer()
         {
             string serverLaunchMutexName = $@"Global\msbuild-server-launch-{_handshake.ComputeHash()}";
+
             try
             {
                 // For unknown root cause, opening mutex can sometimes throw 'Connection timed out' exception. See: https://github.com/dotnet/msbuild/issues/7993
                 using var serverLaunchMutex = ServerNamedMutex.OpenOrCreateMutex(serverLaunchMutexName, out bool mutexCreatedNew);
+
                 if (!mutexCreatedNew)
                 {
                     // Some other client process launching a server and setting a build request for it. Fallback to usual msbuild app build.
                     CommunicationsUtilities.Trace("Another process launching the msbuild server, falling back to former behavior.");
                     _exitResult.MSBuildClientExitType = MSBuildClientExitType.ServerBusy;
                     return false;
                 }
+            }
+            catch (IOException ex) when (ex is not PathTooLongException)
+            {
+                CommunicationsUtilities.Trace("Failed to obtain the current build server state: {0}",  ex);
+                CommunicationsUtilities.Trace("HResult: {0}.", ex.HResult);
+                _exitResult.MSBuildClientExitType = MSBuildClientExitType.UnknownServerState;
+                return false;
+            }
 
+            try
+            {
                 string[] msBuildServerOptions = new string[] {
                     "/nologo",
                     "/nodemode:8"
                 };
-
                 NodeLauncher nodeLauncher = new NodeLauncher();
                 CommunicationsUtilities.Trace("Starting Server...");
                 Process msbuildProcess = nodeLauncher.Start(_msbuildLocation, string.Join(" ", msBuildServerOptions));

src/Build/BackEnd/Client/MSBuildClientExitType.cs

@@ -24,6 +24,14 @@ public enum MSBuildClientExitType
         /// The build stopped unexpectedly, for example,
         /// because a named pipe between the server and the client was unexpectedly closed.
         /// </summary>
-        Unexpected
+        Unexpected,
+        /// <summary>
+        /// The client is not able to identify the server state.
+        /// </summary>
+        /// <remarks>
+        /// This may happen when mutex that is regulating the server state throws.
+        /// See: https://github.com/dotnet/msbuild/issues/7993.
+        /// </remarks>
+        UnknownServerState
     }
 }

src/MSBuild/MSBuildClientApp.cs

@@ -76,6 +76,7 @@ public static MSBuildApp.ExitType Execute(
 
             if (exitResult.MSBuildClientExitType == MSBuildClientExitType.ServerBusy ||
                 exitResult.MSBuildClientExitType == MSBuildClientExitType.UnableToConnect ||
+                exitResult.MSBuildClientExitType == MSBuildClientExitType.UnknownServerState ||
                 exitResult.MSBuildClientExitType == MSBuildClientExitType.LaunchError)
             {
                 if (KnownTelemetry.BuildTelemetry != null)