Fix timers, tiered compilation, introduced time-sensitive work item q…

…ueue to simulate coreclr behavior

src/coreclr/src/System.Private.CoreLib/src/System/Threading/ThreadPool.CoreCLR.cs

@@ -134,27 +134,30 @@ public bool Unregister(WaitHandle waitObject)
         private static extern bool UnregisterWaitNative(IntPtr handle, SafeHandle? waitObject);
     }
 
-    public static partial class ThreadPool
+    internal sealed class UnmanagedThreadPoolWorkItem : IThreadPoolWorkItem
     {
-        // Time in ms for which ThreadPoolWorkQueue.Dispatch keeps executing work items before returning to the OS
-        private const uint DispatchQuantum = 30;
+        private readonly IntPtr _callback;
+        private readonly IntPtr _state;
+
+        public UnmanagedThreadPoolWorkItem(IntPtr callback, IntPtr state)
+        {
+            _callback = callback;
+            _state = state;
+        }
+
+        void IThreadPoolWorkItem.Execute() => ExecuteUnmanagedThreadPoolWorkItem(_callback, _state);
 
+        [DllImport(RuntimeHelpers.QCall, CharSet = CharSet.Unicode)]
+        private static extern void ExecuteUnmanagedThreadPoolWorkItem(IntPtr callback, IntPtr state);
+    }
+
+    public static partial class ThreadPool
+    {
         internal static readonly bool UsePortableThreadPool = InitializeConfigAndDetermineUsePortableThreadPool();
+        internal static bool SupportsTimeSensitiveWorkItems => UsePortableThreadPool;
 
         internal static readonly bool EnableWorkerTracking = GetEnableWorkerTracking();
 
-        internal static bool KeepDispatching(int startTickCount)
-        {
-            if (UsePortableThreadPool)
-            {
-                return true;
-            }
-
-            // Note: this function may incorrectly return false due to TickCount overflow
-            // if work item execution took around a multiple of 2^32 milliseconds (~49.7 days),
-            // which is improbable.
-            return (uint)(Environment.TickCount - startTickCount) < DispatchQuantum;
-        }
 
         private static unsafe bool InitializeConfigAndDetermineUsePortableThreadPool()
         {
@@ -346,6 +349,13 @@ internal static void RequestWorkerThread()
         public static unsafe bool UnsafeQueueNativeOverlapped(NativeOverlapped* overlapped) =>
             PostQueuedCompletionStatus(overlapped);
 
+        // Entry point from unmanaged code
+        private static void UnsafeQueueUnmanagedWorkItem(IntPtr callback, IntPtr state)
+        {
+            Debug.Assert(SupportsTimeSensitiveWorkItems);
+            UnsafeQueueTimeSensitiveWorkItemInternal(new UnmanagedThreadPoolWorkItem(callback, state));
+        }
+
         // Native methods:
 
         [MethodImpl(MethodImplOptions.InternalCall)]
@@ -394,7 +404,7 @@ internal static void NotifyWorkItemProgress()
         {
             if (UsePortableThreadPool)
             {
-                PortableThreadPool.ThreadPoolInstance.NotifyWorkItemComplete();
+                PortableThreadPool.ThreadPoolInstance.NotifyWorkItemProgress();
                 return;
             }
 
@@ -446,4 +456,5 @@ public static bool BindHandle(SafeHandle osHandle)
         [MethodImpl(MethodImplOptions.InternalCall)]
         private static extern bool BindIOCompletionCallbackNative(IntPtr fileHandle);
     }
+
 }

src/coreclr/src/vm/comthreadpool.cpp

@@ -613,6 +613,18 @@ FCIMPLEND
 
 /********************************************************************************************************************/
 
+void QCALLTYPE ThreadPoolNative::ExecuteUnmanagedThreadPoolWorkItem(LPTHREAD_START_ROUTINE callback, LPVOID state)
+{
+    QCALL_CONTRACT;
+
+    BEGIN_QCALL;
+
+    _ASSERTE(ThreadpoolMgr::UsePortableThreadPool());
+    callback(state);
+
+    END_QCALL;
+}
+
 /********************************************************************************************************************/
 
 struct BindIoCompletion_Args

src/coreclr/src/vm/comthreadpool.h

@@ -57,6 +57,8 @@ class ThreadPoolNative
     static FCDECL2(FC_BOOL_RET, CorUnregisterWait, LPVOID WaitHandle, Object * objectToNotify);
     static FCDECL1(void, CorWaitHandleCleanupNative, LPVOID WaitHandle);
     static FCDECL1(FC_BOOL_RET, CorBindIoCompletionCallback, HANDLE fileHandle);
+
+    static void QCALLTYPE ExecuteUnmanagedThreadPoolWorkItem(LPTHREAD_START_ROUTINE callback, LPVOID state);
 };
 
 class AppDomainTimerNative

src/coreclr/src/vm/corelib.h

@@ -905,6 +905,9 @@ DEFINE_METHOD(TP_WAIT_CALLBACK,        PERFORM_WAIT_CALLBACK,               Perf
 DEFINE_CLASS(TIMER_QUEUE,           Threading,                TimerQueue)
 DEFINE_METHOD(TIMER_QUEUE,          APPDOMAIN_TIMER_CALLBACK, AppDomainTimerCallback,   SM_Int_RetVoid)
 
+DEFINE_CLASS(THREAD_POOL,           Threading,                          ThreadPool)
+DEFINE_METHOD(THREAD_POOL,          UNSAFE_QUEUE_UNMANAGED_WORK_ITEM,   UnsafeQueueUnmanagedWorkItem,   SM_IntPtr_IntPtr_RetVoid)
+
 DEFINE_CLASS(TIMESPAN,              System,                 TimeSpan)
 
 

src/coreclr/src/vm/ecalllist.h

@@ -656,12 +656,15 @@ FCFuncStart(gTimerFuncs)
     QCFuncElement("DeleteAppDomainTimer", AppDomainTimerNative::DeleteAppDomainTimer)
 FCFuncEnd()
 
-
 FCFuncStart(gRegisteredWaitHandleFuncs)
     FCFuncElement("UnregisterWaitNative", ThreadPoolNative::CorUnregisterWait)
     FCFuncElement("WaitHandleCleanupNative", ThreadPoolNative::CorWaitHandleCleanupNative)
 FCFuncEnd()
 
+FCFuncStart(gUnmanagedThreadPoolWorkItemFuncs)
+    QCFuncElement("ExecuteUnmanagedThreadPoolWorkItem", ThreadPoolNative::ExecuteUnmanagedThreadPoolWorkItem)
+FCFuncEnd()
+
 FCFuncStart(gWaitHandleFuncs)
     FCFuncElement("WaitOneCore", WaitHandleNative::CorWaitOneNative)
     FCFuncElement("WaitMultipleIgnoringSyncContext", WaitHandleNative::CorWaitMultipleNative)
@@ -1214,6 +1217,7 @@ FCClassElement("TypeBuilder", "System.Reflection.Emit", gCOMClassWriter)
 FCClassElement("TypeLoadException", "System", gTypeLoadExceptionFuncs)
 FCClassElement("TypeNameParser", "System", gTypeNameParser)
 FCClassElement("TypedReference", "System", gTypedReferenceFuncs)
+FCClassElement("UnmanagedThreadPoolWorkItem", "System.Threading", gUnmanagedThreadPoolWorkItemFuncs)
 #ifdef FEATURE_UTF8STRING
 FCClassElement("Utf8String", "System", gUtf8StringFuncs)
 #endif // FEATURE_UTF8STRING

src/coreclr/src/vm/tieredcompilation.cpp

@@ -93,6 +93,7 @@ TieredCompilationManager::TieredCompilationManager() :
     m_countOfNewMethodsCalledDuringDelay(0),
     m_methodsPendingCountingForTier1(nullptr),
     m_tieringDelayTimerHandle(nullptr),
+    m_doBackgroundWorkTimerHandle(nullptr),
     m_isBackgroundWorkScheduled(false),
     m_tier1CallCountingCandidateMethodRecentlyRecorded(false),
     m_isPendingCallCountingCompletion(false),
@@ -566,17 +567,65 @@ void TieredCompilationManager::RequestBackgroundWork()
     WRAPPER_NO_CONTRACT;
     _ASSERTE(m_isBackgroundWorkScheduled);
 
+    if (ThreadpoolMgr::UsePortableThreadPool())
+    {
+        // QueueUserWorkItem is not intended to be supported in this mode, and there are call sites of this function where
+        // managed code cannot be called instead to queue a work item. Use a timer with zero due time instead, which would on
+        // the timer thread call into managed code to queue a work item.
+
+        NewHolder<ThreadpoolMgr::TimerInfoContext> timerContextHolder = new ThreadpoolMgr::TimerInfoContext();
+        timerContextHolder->TimerId = 0;
+
+        _ASSERTE(m_doBackgroundWorkTimerHandle == nullptr);
+        if (!ThreadpoolMgr::CreateTimerQueueTimer(
+                &m_doBackgroundWorkTimerHandle,
+                DoBackgroundWorkTimerCallback,
+                timerContextHolder,
+                0 /* DueTime */,
+                (DWORD)-1 /* Period, non-repeating */,
+                0 /* Flags */))
+        {
+            _ASSERTE(m_doBackgroundWorkTimerHandle == nullptr);
+            ThrowOutOfMemory();
+        }
+
+        timerContextHolder.SuppressRelease(); // the timer context is automatically deleted by the timer infrastructure
+        return;
+    }
+
     if (!ThreadpoolMgr::QueueUserWorkItem(StaticBackgroundWorkCallback, this, QUEUE_ONLY, TRUE))
     {
         ThrowOutOfMemory();
     }
 }
 
+void WINAPI TieredCompilationManager::DoBackgroundWorkTimerCallback(PVOID parameter, BOOLEAN timerFired)
+{
+    CONTRACTL
+    {
+        THROWS;
+        GC_TRIGGERS;
+        MODE_PREEMPTIVE;
+    }
+    CONTRACTL_END;
+
+    _ASSERTE(ThreadpoolMgr::UsePortableThreadPool());
+    _ASSERTE(timerFired);
+
+    TieredCompilationManager *pTieredCompilationManager = GetAppDomain()->GetTieredCompilationManager();
+    _ASSERTE(pTieredCompilationManager->m_doBackgroundWorkTimerHandle != nullptr);
+    ThreadpoolMgr::DeleteTimerQueueTimer(pTieredCompilationManager->m_doBackgroundWorkTimerHandle, nullptr);
+    pTieredCompilationManager->m_doBackgroundWorkTimerHandle = nullptr;
+
+    pTieredCompilationManager->DoBackgroundWork();
+}
+
 // This is the initial entrypoint for the background thread, called by
 // the threadpool.
 DWORD WINAPI TieredCompilationManager::StaticBackgroundWorkCallback(void *args)
 {
     STANDARD_VM_CONTRACT;
+    _ASSERTE(!ThreadpoolMgr::UsePortableThreadPool());
 
     TieredCompilationManager * pTieredCompilationManager = (TieredCompilationManager *)args;
     pTieredCompilationManager->DoBackgroundWork();
@@ -590,6 +639,7 @@ DWORD WINAPI TieredCompilationManager::StaticBackgroundWorkCallback(void *args)
 void TieredCompilationManager::DoBackgroundWork()
 {
     WRAPPER_NO_CONTRACT;
+    _ASSERTE(m_doBackgroundWorkTimerHandle == nullptr);
 
     AutoResetIsBackgroundWorkScheduled autoResetIsBackgroundWorkScheduled(this);
 

src/coreclr/src/vm/tieredcompilation.h

@@ -57,6 +57,7 @@ class TieredCompilationManager
     void ScheduleBackgroundWork();
 private:
     void RequestBackgroundWork();
+    static void WINAPI DoBackgroundWorkTimerCallback(PVOID parameter, BOOLEAN timerFired);
     static DWORD StaticBackgroundWorkCallback(void* args);
     void DoBackgroundWork();
 
@@ -109,13 +110,12 @@ class TieredCompilationManager
     UINT32 m_countOfNewMethodsCalledDuringDelay;
     SArray<MethodDesc*>* m_methodsPendingCountingForTier1;
     HANDLE m_tieringDelayTimerHandle;
+    HANDLE m_doBackgroundWorkTimerHandle;
     bool m_isBackgroundWorkScheduled;
     bool m_tier1CallCountingCandidateMethodRecentlyRecorded;
     bool m_isPendingCallCountingCompletion;
     bool m_recentlyRequestedCallCountingCompletionAgain;
 
-    CLREvent m_asyncWorkDoneEvent;
-
 #endif // FEATURE_TIERED_COMPILATION
 };
 

src/coreclr/src/vm/win32threadpool.cpp

@@ -4492,6 +4492,7 @@ BOOL ThreadpoolMgr::CreateTimerQueueTimer(PHANDLE phNewTimer,
             if (!params.setupSucceeded)
             {
                 CloseHandle(TimerThreadHandle);
+                *phNewTimer = NULL;
                 return FALSE;
             }
 
@@ -4503,8 +4504,6 @@ BOOL ThreadpoolMgr::CreateTimerQueueTimer(PHANDLE phNewTimer,
 
     NewHolder<TimerInfo> timerInfoHolder;
     TimerInfo * timerInfo = new (nothrow) TimerInfo;
-    *phNewTimer = (HANDLE) timerInfo;
-
     if (NULL == timerInfo)
         ThrowOutOfMemory();
 
@@ -4519,9 +4518,12 @@ BOOL ThreadpoolMgr::CreateTimerQueueTimer(PHANDLE phNewTimer,
     timerInfo->ExternalCompletionEvent = INVALID_HANDLE;
     timerInfo->ExternalEventSafeHandle = NULL;
 
+    *phNewTimer = (HANDLE)timerInfo;
+
     BOOL status = QueueUserAPC((PAPCFUNC)InsertNewTimer,TimerThread,(size_t)timerInfo);
     if (FALSE == status)
     {
+        *phNewTimer = NULL;
         return FALSE;
     }
 
@@ -4706,9 +4708,19 @@ DWORD ThreadpoolMgr::FireTimers()
 
                 InterlockedIncrement(&timerInfo->refCount);
 
-                QueueUserWorkItem(AsyncTimerCallbackCompletion,
-                                  timerInfo,
-                                  QUEUE_ONLY /* TimerInfo take care of deleting*/);
+                if (UsePortableThreadPool())
+                {
+                    GCX_COOP();
+
+                    ARG_SLOT args[] = { PtrToArgSlot(AsyncTimerCallbackCompletion), PtrToArgSlot(timerInfo) };
+                    MethodDescCallSite(METHOD__THREAD_POOL__UNSAFE_QUEUE_UNMANAGED_WORK_ITEM).Call(args);
+                }
+                else
+                {
+                    QueueUserWorkItem(AsyncTimerCallbackCompletion,
+                                      timerInfo,
+                                      QUEUE_ONLY /* TimerInfo take care of deleting*/);
+                }
 
                 if (timerInfo->Period != 0 && timerInfo->Period != (ULONG)-1)
                 {

src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.cs

@@ -195,7 +195,7 @@ public int GetAvailableThreads()
         public int ThreadCount => ThreadCounts.VolatileReadCounts(ref _separated.counts).numExistingThreads;
         public long CompletedWorkItemCount => _completionCounter.Count;
 
-        internal bool NotifyWorkItemComplete()
+        internal void NotifyWorkItemProgress()
         {
             _completionCounter.Increment();
             Volatile.Write(ref _separated.lastDequeueTime, Environment.TickCount);
@@ -211,7 +211,11 @@ internal bool NotifyWorkItemComplete()
                     _hillClimbingThreadAdjustmentLock.Release();
                 }
             }
+        }
 
+        internal bool NotifyWorkItemComplete()
+        {
+            NotifyWorkItemProgress();
             return !WorkerThread.ShouldStopProcessingWorkNow();
         }
 

src/libraries/System.Private.CoreLib/src/System/Threading/ThreadPool.Portable.cs

@@ -25,9 +25,9 @@ public sealed partial class RegisteredWaitHandle : MarshalByRefObject
 
     public static partial class ThreadPool
     {
+        internal const bool SupportsTimeSensitiveWorkItems = true;
         internal const bool EnableWorkerTracking = false;
 
-
         public static bool SetMaxThreads(int workerThreads, int completionPortThreads)
         {
             if (workerThreads < 0 || completionPortThreads < 0)
@@ -90,14 +90,9 @@ internal static void RequestWorkerThread()
             PortableThreadPool.ThreadPoolInstance.RequestWorker();
         }
 
-        internal static bool KeepDispatching(int startTickCount)
-        {
-            return true;
-        }
-
         internal static void NotifyWorkItemProgress()
         {
-            PortableThreadPool.ThreadPoolInstance.NotifyWorkItemComplete();
+            PortableThreadPool.ThreadPoolInstance.NotifyWorkItemProgress();
         }
 
         internal static bool NotifyWorkItemComplete()