Miscellaneous perf fixes

src/coreclr/src/System.Private.CoreLib/src/System/Threading/ThreadPool.CoreCLR.cs

@@ -423,11 +423,14 @@ private static void UnsafeQueueUnmanagedWorkItem(IntPtr callback, IntPtr state)
         private static extern void GetAvailableThreadsNative(out int workerThreads, out int completionPortThreads);
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static bool NotifyWorkItemComplete(object? threadLocalCompletionCountObject)
+        internal static bool NotifyWorkItemComplete(object? threadLocalCompletionCountObject, int currentTimeMs)
         {
             if (UsePortableThreadPool)
             {
-                return PortableThreadPool.ThreadPoolInstance.NotifyWorkItemComplete(threadLocalCompletionCountObject);
+                return
+                    PortableThreadPool.ThreadPoolInstance.NotifyWorkItemComplete(
+                        threadLocalCompletionCountObject,
+                        currentTimeMs);
             }
 
             return NotifyWorkItemCompleteNative();

src/libraries/System.Private.CoreLib/src/System/Collections/Concurrent/ConcurrentQueue.cs

@@ -666,9 +666,28 @@ private void EnqueueSlow(T item)
         /// true if an element was removed and returned from the beginning of the
         /// <see cref="ConcurrentQueue{T}"/> successfully; otherwise, false.
         /// </returns>
-        public bool TryDequeue([MaybeNullWhen(false)] out T result) =>
-            _head.TryDequeue(out result) || // fast-path that operates just on the head segment
-            TryDequeueSlow(out result); // slow path that needs to fix up segments
+        public bool TryDequeue([MaybeNullWhen(false)] out T result)
+        {
+            // Get the current head
+            ConcurrentQueueSegment<T> head = _head;
+
+            // Try to take.  If we're successful, we're done.
+            if (head.TryDequeue(out result))
+            {
+                return true;
+            }
+
+            // Check to see whether this segment is the last. If it is, we can consider
+            // this to be a moment-in-time empty condition (even though between the TryDequeue
+            // check and this check, another item could have arrived).
+            if (head._nextSegment == null)
+            {
+                result = default!;
+                return false;
+            }
+
+            return TryDequeueSlow(out result); // slow path that needs to fix up segments
+        }
 
         /// <summary>Tries to dequeue an item, removing empty segments as needed.</summary>
         private bool TryDequeueSlow([MaybeNullWhen(false)] out T item)

src/libraries/System.Private.CoreLib/src/System/Threading/LowLevelLifoSemaphore.cs

@@ -16,10 +16,11 @@ internal sealed partial class LowLevelLifoSemaphore : IDisposable
 
         private readonly int _maximumSignalCount;
         private readonly int _spinCount;
+        private readonly Action _onWait;
 
         private const int SpinSleep0Threshold = 10;
 
-        public LowLevelLifoSemaphore(int initialSignalCount, int maximumSignalCount, int spinCount)
+        public LowLevelLifoSemaphore(int initialSignalCount, int maximumSignalCount, int spinCount, Action onWait)
         {
             Debug.Assert(initialSignalCount >= 0);
             Debug.Assert(initialSignalCount <= maximumSignalCount);
@@ -30,6 +31,7 @@ public LowLevelLifoSemaphore(int initialSignalCount, int maximumSignalCount, int
             _separated._counts.SignalCount = (uint)initialSignalCount;
             _maximumSignalCount = maximumSignalCount;
             _spinCount = spinCount;
+            _onWait = onWait;
 
             Create(maximumSignalCount);
         }
@@ -38,6 +40,8 @@ public bool Wait(int timeoutMs)
         {
             Debug.Assert(timeoutMs >= -1);
 
+            int spinCount = _spinCount;
+
             // Try to acquire the semaphore or
             // a) register as a spinner if spinCount > 0 and timeoutMs > 0
             // b) register as a waiter if there's already too many spinners or spinCount == 0 and timeoutMs > 0
@@ -53,7 +57,7 @@ public bool Wait(int timeoutMs)
                 }
                 else if (timeoutMs != 0)
                 {
-                    if (_spinCount > 0 && newCounts.SpinnerCount < byte.MaxValue)
+                    if (spinCount > 0 && newCounts.SpinnerCount < byte.MaxValue)
                     {
                         newCounts.IncrementSpinnerCount();
                     }
@@ -85,9 +89,13 @@ public bool Wait(int timeoutMs)
                 counts = countsBeforeUpdate;
             }
 
+#if CORECLR && TARGET_UNIX
+            // The PAL's wait subsystem is slower, spin more to compensate for the more expensive wait
+            spinCount *= 2;
+#endif
             int processorCount = Environment.ProcessorCount;
             int spinIndex = processorCount > 1 ? 0 : SpinSleep0Threshold;
-            while (spinIndex < _spinCount)
+            while (spinIndex < spinCount)
             {
                 LowLevelSpinWaiter.Wait(spinIndex, SpinSleep0Threshold, processorCount);
                 spinIndex++;
@@ -189,6 +197,8 @@ private bool WaitForSignal(int timeoutMs)
         {
             Debug.Assert(timeoutMs > 0 || timeoutMs == -1);
 
+            _onWait();
+
             while (true)
             {
                 if (!WaitCore(timeoutMs))

src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.GateThread.cs

@@ -2,6 +2,8 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 
 namespace System.Threading
 {
@@ -13,8 +15,6 @@ private static class GateThread
             private const int DequeueDelayThresholdMs = GateThreadDelayMs * 2;
             private const int GateThreadRunningMask = 0x4;
 
-            private static int s_runningState;
-
             private static readonly AutoResetEvent s_runGateThreadEvent = new AutoResetEvent(initialState: true);
 
             private const int MaxRuns = 2;
@@ -32,6 +32,9 @@ private static void GateThreadStart()
                 CpuUtilizationReader cpuUtilizationReader = default;
                 _ = cpuUtilizationReader.CurrentUtilization;
 
+                PortableThreadPool threadPoolInstance = ThreadPoolInstance;
+                LowLevelLock hillClimbingThreadAdjustmentLock = threadPoolInstance._hillClimbingThreadAdjustmentLock;
+
                 while (true)
                 {
                     s_runGateThreadEvent.WaitOne();
@@ -42,20 +45,20 @@ private static void GateThreadStart()
                         Thread.Sleep(GateThreadDelayMs);
 
                         int cpuUtilization = cpuUtilizationReader.CurrentUtilization;
-                        ThreadPoolInstance._cpuUtilization = cpuUtilization;
+                        threadPoolInstance._cpuUtilization = cpuUtilization;
 
                         needGateThreadForRuntime = ThreadPool.PerformRuntimeSpecificGateActivities(cpuUtilization);
 
                         if (!disableStarvationDetection &&
-                            ThreadPoolInstance._separated.numRequestedWorkers > 0 &&
-                            SufficientDelaySinceLastDequeue())
+                            threadPoolInstance._separated.numRequestedWorkers > 0 &&
+                            SufficientDelaySinceLastDequeue(threadPoolInstance))
                         {
                             try
                             {
-                                ThreadPoolInstance._hillClimbingThreadAdjustmentLock.Acquire();
-                                ThreadCounts counts = ThreadPoolInstance._separated.counts.VolatileRead();
+                                hillClimbingThreadAdjustmentLock.Acquire();
+                                ThreadCounts counts = threadPoolInstance._separated.counts.VolatileRead();
                                 // don't add a thread if we're at max or if we are already in the process of adding threads
-                                while (counts.NumExistingThreads < ThreadPoolInstance._maxThreads && counts.NumExistingThreads >= counts.NumThreadsGoal)
+                                while (counts.NumExistingThreads < threadPoolInstance._maxThreads && counts.NumExistingThreads >= counts.NumThreadsGoal)
                                 {
                                     if (debuggerBreakOnWorkStarvation)
                                     {
@@ -65,11 +68,11 @@ private static void GateThreadStart()
                                     ThreadCounts newCounts = counts;
                                     short newNumThreadsGoal = (short)(newCounts.NumExistingThreads + 1);
                                     newCounts.NumThreadsGoal = newNumThreadsGoal;
-                                    ThreadCounts oldCounts = ThreadPoolInstance._separated.counts.InterlockedCompareExchange(newCounts, counts);
+                                    ThreadCounts oldCounts = threadPoolInstance._separated.counts.InterlockedCompareExchange(newCounts, counts);
                                     if (oldCounts == counts)
                                     {
                                         HillClimbing.ThreadPoolHillClimber.ForceChange(newNumThreadsGoal, HillClimbing.StateOrTransition.Starvation);
-                                        WorkerThread.MaybeAddWorkingWorker();
+                                        WorkerThread.MaybeAddWorkingWorker(threadPoolInstance);
                                         break;
                                     }
 
@@ -78,62 +81,61 @@ private static void GateThreadStart()
                             }
                             finally
                             {
-                                ThreadPoolInstance._hillClimbingThreadAdjustmentLock.Release();
+                                hillClimbingThreadAdjustmentLock.Release();
                             }
                         }
                     } while (
                         needGateThreadForRuntime ||
-                        ThreadPoolInstance._separated.numRequestedWorkers > 0 ||
-                        Interlocked.Decrement(ref s_runningState) > GetRunningStateForNumRuns(0));
+                        threadPoolInstance._separated.numRequestedWorkers > 0 ||
+                        Interlocked.Decrement(ref threadPoolInstance._separated.gateThreadRunningState) > GetRunningStateForNumRuns(0));
                 }
             }
 
             // called by logic to spawn new worker threads, return true if it's been too long
             // since the last dequeue operation - takes number of worker threads into account
             // in deciding "too long"
-            private static bool SufficientDelaySinceLastDequeue()
+            private static bool SufficientDelaySinceLastDequeue(PortableThreadPool threadPoolInstance)
             {
-                int delay = Environment.TickCount - Volatile.Read(ref ThreadPoolInstance._separated.lastDequeueTime);
+                int delay = Environment.TickCount - Volatile.Read(ref threadPoolInstance._separated.lastDequeueTime);
 
                 int minimumDelay;
 
-                if (ThreadPoolInstance._cpuUtilization < CpuUtilizationLow)
+                if (threadPoolInstance._cpuUtilization < CpuUtilizationLow)
                 {
                     minimumDelay = GateThreadDelayMs;
                 }
                 else
                 {
-                    ThreadCounts counts = ThreadPoolInstance._separated.counts.VolatileRead();
+                    ThreadCounts counts = threadPoolInstance._separated.counts.VolatileRead();
                     int numThreads = counts.NumThreadsGoal;
                     minimumDelay = numThreads * DequeueDelayThresholdMs;
                 }
                 return delay > minimumDelay;
             }
 
             // This is called by a worker thread
-            internal static void EnsureRunning()
+            internal static void EnsureRunning(PortableThreadPool threadPoolInstance)
             {
-                int numRunsMask = Interlocked.Exchange(ref s_runningState, GetRunningStateForNumRuns(MaxRuns));
-                if ((numRunsMask & GateThreadRunningMask) == 0)
+                // The callers ensure that this speculative load is sufficient to ensure that the gate thread is activated when
+                // it is needed
+                if (threadPoolInstance._separated.gateThreadRunningState != GetRunningStateForNumRuns(MaxRuns))
                 {
-                    bool created = false;
-                    try
-                    {
-                        CreateGateThread();
-                        created = true;
-                    }
-                    finally
-                    {
-                        if (!created)
-                        {
-                            Interlocked.Exchange(ref s_runningState, 0);
-                        }
-                    }
+                    EnsureRunningSlow(threadPoolInstance);
                 }
-                else if (numRunsMask == GetRunningStateForNumRuns(0))
+            }
+
+            [MethodImpl(MethodImplOptions.NoInlining)]
+            internal static void EnsureRunningSlow(PortableThreadPool threadPoolInstance)
+            {
+                int numRunsMask = Interlocked.Exchange(ref threadPoolInstance._separated.gateThreadRunningState, GetRunningStateForNumRuns(MaxRuns));
+                if (numRunsMask == GetRunningStateForNumRuns(0))
                 {
                     s_runGateThreadEvent.Set();
                 }
+                else if ((numRunsMask & GateThreadRunningMask) == 0)
+                {
+                    CreateGateThread(threadPoolInstance);
+                }
             }
 
             private static int GetRunningStateForNumRuns(int numRuns)
@@ -143,16 +145,29 @@ private static int GetRunningStateForNumRuns(int numRuns)
                 return GateThreadRunningMask | numRuns;
             }
 
-            private static void CreateGateThread()
+            [MethodImpl(MethodImplOptions.NoInlining)]
+            private static void CreateGateThread(PortableThreadPool threadPoolInstance)
             {
-                Thread gateThread = new Thread(GateThreadStart);
-                gateThread.IsThreadPoolThread = true;
-                gateThread.IsBackground = true;
-                gateThread.Name = ".NET ThreadPool Gate";
-                gateThread.Start();
+                bool created = false;
+                try
+                {
+                    Thread gateThread = new Thread(GateThreadStart);
+                    gateThread.IsThreadPoolThread = true;
+                    gateThread.IsBackground = true;
+                    gateThread.Name = ".NET ThreadPool Gate";
+                    gateThread.Start();
+                    created = true;
+                }
+                finally
+                {
+                    if (!created)
+                    {
+                        Interlocked.Exchange(ref threadPoolInstance._separated.gateThreadRunningState, 0);
+                    }
+                }
             }
         }
 
-        internal static void EnsureGateThreadRunning() => GateThread.EnsureRunning();
+        internal static void EnsureGateThreadRunning() => GateThread.EnsureRunning(ThreadPoolInstance);
     }
 }

src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.HillClimbing.cs

@@ -320,7 +320,8 @@ public HillClimbing(int wavePeriod, int maxWaveMagnitude, double waveMagnitudeMu
                 //
                 // If the result was positive, and CPU is > 95%, refuse the move.
                 //
-                if (move > 0.0 && ThreadPoolInstance._cpuUtilization > CpuUtilizationHigh)
+                PortableThreadPool threadPoolInstance = ThreadPoolInstance;
+                if (move > 0.0 && threadPoolInstance._cpuUtilization > CpuUtilizationHigh)
                     move = 0.0;
 
                 //
@@ -339,8 +340,8 @@ public HillClimbing(int wavePeriod, int maxWaveMagnitude, double waveMagnitudeMu
                 //
                 // Make sure our control setting is within the ThreadPool's limits
                 //
-                int maxThreads = ThreadPoolInstance._maxThreads;
-                int minThreads = ThreadPoolInstance._minThreads;
+                int maxThreads = threadPoolInstance._maxThreads;
+                int minThreads = threadPoolInstance._minThreads;
 
                 _currentControlSetting = Math.Min(maxThreads - newThreadWaveMagnitude, _currentControlSetting);
                 _currentControlSetting = Math.Max(minThreads, _currentControlSetting);

src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WaitThread.cs

@@ -275,7 +275,8 @@ private void WaitThreadStart()
             /// </summary>
             private void ProcessRemovals()
             {
-                ThreadPoolInstance._waitThreadLock.Acquire();
+                PortableThreadPool threadPoolInstance = ThreadPoolInstance;
+                threadPoolInstance._waitThreadLock.Acquire();
                 try
                 {
                     Debug.Assert(_numPendingRemoves >= 0);
@@ -322,7 +323,7 @@ private void ProcessRemovals()
                 }
                 finally
                 {
-                    ThreadPoolInstance._waitThreadLock.Release();
+                    threadPoolInstance._waitThreadLock.Release();
                 }
             }
 
@@ -398,7 +399,8 @@ private void UnregisterWait(RegisteredWaitHandle handle, bool blocking)
             {
                 bool pendingRemoval = false;
                 // TODO: Optimization: Try to unregister wait directly if it isn't being waited on.
-                ThreadPoolInstance._waitThreadLock.Acquire();
+                PortableThreadPool threadPoolInstance = ThreadPoolInstance;
+                threadPoolInstance._waitThreadLock.Acquire();
                 try
                 {
                     // If this handle is not already pending removal and hasn't already been removed
@@ -411,7 +413,7 @@ private void UnregisterWait(RegisteredWaitHandle handle, bool blocking)
                 }
                 finally
                 {
-                    ThreadPoolInstance._waitThreadLock.Release();
+                    threadPoolInstance._waitThreadLock.Release();
                 }
 
                 if (blocking)