R2R -> InstrumentedTier0 for hot R2R methods

src/coreclr/inc/clrconfigvalues.h

@@ -572,11 +572,16 @@ RETAIL_CONFIG_DWORD_INFO(INTERNAL_HillClimbing_GainExponent,
 #define TC_CallCountThreshold (2)
 #define TC_CallCountingDelayMs (1)
 #define TC_DelaySingleProcMultiplier (2)
+#define TC_DelayPgoDivider (1)
 #else // !_DEBUG
 #define TC_BackgroundWorkerTimeoutMs (4000)
-#define TC_CallCountThreshold (30)
+#define TC_CallCountThreshold (30) // TODO: Find better defaults for PGO
 #define TC_CallCountingDelayMs (100)
 #define TC_DelaySingleProcMultiplier (10)
+#define TC_DelayPgoDivider (3) // in PGO mode we compile way more methods and don't want to stuck in tier0
+                               // waiting for a Delay tick to find out it's busy (see IsTieringDelayActive())
+                               // and wait again and again
+
 #endif // _DEBUG
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_TieredCompilation, W("TieredCompilation"), 1, "Enables tiered compilation")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_TC_QuickJit, W("TC_QuickJit"), 1, "For methods that would be jitted, enable using quick JIT when appropriate.")
@@ -590,6 +595,7 @@ RETAIL_CONFIG_DWORD_INFO(INTERNAL_TC_BackgroundWorkerTimeoutMs, W("TC_Background
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_TC_CallCountThreshold, W("TC_CallCountThreshold"), TC_CallCountThreshold, "Number of times a method must be called in tier 0 after which it is promoted to the next tier.")
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_TC_CallCountingDelayMs, W("TC_CallCountingDelayMs"), TC_CallCountingDelayMs, "A perpetual delay in milliseconds that is applied call counting in tier 0 and jitting at higher tiers, while there is startup-like activity.")
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_TC_DelaySingleProcMultiplier, W("TC_DelaySingleProcMultiplier"), TC_DelaySingleProcMultiplier, "Multiplier for TC_CallCountingDelayMs that is applied on a single-processor machine or when the process is affinitized to a single processor.")
+RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_TC_DelayPgoDivider, W("TC_DelayPgoDivider"), TC_DelayPgoDivider, "PGO-specific Divider for TC_CallCountingDelayMs, needed because we jit more methods")
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_TC_CallCounting, W("TC_CallCounting"), 1, "Enabled by default (only activates when TieredCompilation is also enabled). If disabled immediately backpatches prestub, and likely prevents any promotion to higher tiers")
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_TC_UseCallCountingStubs, W("TC_UseCallCountingStubs"), 1, "Uses call counting stubs for faster call counting.")
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_TC_DeleteCallCountingStubsAfter, W("TC_DeleteCallCountingStubsAfter"), 0, "Deletes call counting stubs after this many have completed. Zero to disable deleting.")
@@ -617,7 +623,9 @@ CONFIG_DWORD_INFO(INTERNAL_OSR_HighId, W("OSR_HighId"), 10000000, "High end of e
 RETAIL_CONFIG_STRING_INFO(INTERNAL_PGODataPath, W("PGODataPath"), "Read/Write PGO data from/to the indicated file.")
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_ReadPGOData, W("ReadPGOData"), 0, "Read PGO data")
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_WritePGOData, W("WritePGOData"), 0, "Write PGO data")
-RETAIL_CONFIG_DWORD_INFO(INTERNAL_TieredPGO, W("TieredPGO"), 0, "Instrument Tier0 code and make counts available to Tier1")
+
+// Enable for CI tests
+RETAIL_CONFIG_DWORD_INFO(INTERNAL_TieredPGO, W("TieredPGO"), 1, "Instrument Tier0 code and make counts available to Tier1")
 #endif
 
 ///

src/coreclr/inc/corjitflags.h

@@ -97,7 +97,7 @@ class CORJIT_FLAGS
         CORJIT_FLAG_UNUSED16                = 43,
 #endif // !defined(TARGET_ARM)
 
-        CORJIT_FLAG_UNUSED17                = 44,
+        CORJIT_FLAG_PROMOTED                = 44, // Method was promoted at least once and is likely hot
         CORJIT_FLAG_UNUSED18                = 45,
         CORJIT_FLAG_UNUSED19                = 46,
         CORJIT_FLAG_UNUSED20                = 47,

src/coreclr/jit/compiler.cpp

@@ -6668,7 +6668,7 @@ int Compiler::compCompileHelper(CORINFO_MODULE_HANDLE classPtr,
             }
         }
 
-        if (reason != nullptr)
+        if ((reason != nullptr) && !opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PROMOTED))
         {
             fgSwitchToOptimized(reason);
         }

src/coreclr/jit/compiler.hpp

@@ -4759,6 +4759,11 @@ inline bool Compiler::compCanHavePatchpoints(const char** reason)
     {
         whyNot = "OSR can't handle reverse pinvoke";
     }
+    else if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PROMOTED))
+    {
+        // This method will likely make it to tier1 on its own soon
+        whyNot = "OSR is not needed in an already promoted tier";
+    }
 #else
     whyNot = "OSR feature not defined in build";
 #endif

src/coreclr/jit/fgbasic.cpp

@@ -3568,6 +3568,12 @@ void Compiler::fgCheckForLoopsInHandlers()
         return;
     }
 
+    if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PROMOTED))
+    {
+        // This method will likely make it to tier1 on its own soon
+        return;
+    }
+
     if (info.compXcptnsCount == 0)
     {
         return;

src/coreclr/jit/importer.cpp

@@ -12388,6 +12388,12 @@ void Compiler::impImportBlockCode(BasicBlock* block)
 
     bool enablePatchpoints = opts.jitFlags->IsSet(JitFlags::JIT_FLAG_TIER0) && (JitConfig.TC_OnStackReplacement() > 0);
 
+    if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PROMOTED))
+    {
+        // This method will likely make it to tier1 on its own soon
+        enablePatchpoints = false;
+    }
+
 #ifdef DEBUG
 
     // Optionally suppress patchpoints by method hash

src/coreclr/jit/jitee.h

@@ -87,7 +87,7 @@ class JitFlags
         JIT_FLAG_UNUSED16                = 43,
 #endif // !defined(TARGET_ARM)
 
-        JIT_FLAG_UNUSED17                = 44,
+        JIT_FLAG_PROMOTED                = 44, // Method was promoted at least once and is likely hot
         JIT_FLAG_UNUSED18                = 45,
         JIT_FLAG_UNUSED19                = 46,
         JIT_FLAG_UNUSED20                = 47,

src/coreclr/vm/callcounting.cpp

@@ -147,13 +147,18 @@ FORCEINLINE void CallCountingManager::CallCountingInfo::SetStage(Stage stage)
             --s_activeCallCountingStubCount;
             break;
 
+        case Stage::PendingReset:
+            _ASSERT(m_stage >= Stage::PendingCompletion);
+            --s_activeCallCountingStubCount;
+            break;
+
         case Stage::StubMayBeActive:
             _ASSERTE(m_callCountingStub != nullptr);
             FALLTHROUGH;
 
         case Stage::PendingCompletion:
-            _ASSERTE(m_stage == Stage::StubIsNotActive || m_stage == Stage::StubMayBeActive);
-            if (m_stage == Stage::StubIsNotActive && m_callCountingStub != nullptr)
+            _ASSERTE(m_stage == Stage::StubIsNotActive || m_stage == Stage::StubMayBeActive || m_stage == Stage::PendingReset);
+            if ((m_stage == Stage::StubIsNotActive || m_stage == Stage::PendingReset) && m_callCountingStub != nullptr)
             {
                 ++s_activeCallCountingStubCount;
             }
@@ -178,6 +183,13 @@ FORCEINLINE void CallCountingManager::CallCountingInfo::SetStage(Stage stage)
 
     m_stage = stage;
 }
+
+void CallCountingManager::CallCountingInfo::ReuseForNewVersion(NativeCodeVersion version)
+{
+    WRAPPER_NO_CONTRACT;
+    m_codeVersion = version;
+    SetStage(Stage::PendingReset);
+}
 #endif
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -539,6 +551,28 @@ void CallCountingManager::DisableCallCounting(NativeCodeVersion codeVersion)
     callCountingInfoHolder.SuppressRelease();
 }
 
+void CallCountingManager::ReuseStubForNewVersion(NativeCodeVersion oldVersion, NativeCodeVersion newVersion)
+{
+    CONTRACTL
+    {
+        THROWS;
+        GC_NOTRIGGER;
+        MODE_ANY;
+    }
+    CONTRACTL_END;
+
+    CodeVersionManager::LockHolder codeVersioningLockHolder;
+    CallCountingInfo* callCountingInfo = m_callCountingInfoByCodeVersionHash.Lookup(oldVersion);
+
+    // No problem if it's null, it's just an optimization to allocate less call counting stubs
+    if (callCountingInfo != nullptr)
+    {
+        m_callCountingInfoByCodeVersionHash.Remove(oldVersion);
+        callCountingInfo->ReuseForNewVersion(newVersion);
+        m_callCountingInfoByCodeVersionHash.Add(callCountingInfo);
+    }
+}
+
 // Returns true if the code entry point was updated to reflect the active code version, false otherwise. In normal paths, the
 // code entry point is not updated only when the use of call counting stubs is disabled, as in that case returning to the
 // prestub is necessary for further call counting. On exception, the code entry point may or may not have been updated and it's
@@ -574,7 +608,7 @@ bool CallCountingManager::SetCodeEntryPoint(
             // For a default code version that is not tier 0, call counting will have been disabled by this time (checked
             // below). Avoid the redundant and not-insignificant expense of GetOptimizationTier() on a default code version.
             !activeCodeVersion.IsDefaultVersion() &&
-            activeCodeVersion.GetOptimizationTier() != NativeCodeVersion::OptimizationTier0
+            !activeCodeVersion.IsUnoptimizedTier()
         ) ||
         !g_pConfig->TieredCompilation_CallCounting())
     {
@@ -602,7 +636,7 @@ bool CallCountingManager::SetCodeEntryPoint(
                 return true;
             }
 
-            _ASSERTE(activeCodeVersion.GetOptimizationTier() == NativeCodeVersion::OptimizationTier0);
+            _ASSERTE(activeCodeVersion.IsUnoptimizedTier());
 
             // If the tiering delay is active, postpone further work
             if (GetAppDomain()
@@ -618,6 +652,15 @@ bool CallCountingManager::SetCodeEntryPoint(
 
             do
             {
+                if (callCountingStage == CallCountingInfo::Stage::PendingReset)
+                {
+                    // Consider using a different threshold here
+                    CallCount callCountThreshold = g_pConfig->TieredCompilation_CallCountThreshold();
+                    _ASSERTE(callCountThreshold != 0);
+                    *callCountingInfo->GetRemainingCallCountCell() = callCountThreshold;
+                    break;
+                }
+
                 if (!wasMethodCalled)
                 {
                     break;
@@ -649,7 +692,7 @@ bool CallCountingManager::SetCodeEntryPoint(
         }
         else
         {
-            _ASSERTE(activeCodeVersion.GetOptimizationTier() == NativeCodeVersion::OptimizationTier0);
+            _ASSERTE(activeCodeVersion.IsUnoptimizedTier());
 
             // If the tiering delay is active, postpone further work
             if (GetAppDomain()
@@ -659,7 +702,7 @@ bool CallCountingManager::SetCodeEntryPoint(
                 return true;
             }
 
-            CallCount callCountThreshold = (CallCount)g_pConfig->TieredCompilation_CallCountThreshold();
+            CallCount callCountThreshold = g_pConfig->TieredCompilation_CallCountThreshold();
             _ASSERTE(callCountThreshold != 0);
 
             NewHolder<CallCountingInfo> callCountingInfoHolder = new CallCountingInfo(activeCodeVersion, callCountThreshold);
@@ -780,7 +823,7 @@ PCODE CallCountingManager::OnCallCountThresholdReached(TransitionBlock *transiti
     // used going forward under appropriate locking to synchronize further with deletion.
     GCX_PREEMP_THREAD_EXISTS(CURRENT_THREAD);
 
-    _ASSERTE(codeVersion.GetOptimizationTier() == NativeCodeVersion::OptimizationTier0);
+    _ASSERTE(codeVersion.IsUnoptimizedTier());
 
     codeEntryPoint = codeVersion.GetNativeCode();
     do
@@ -797,7 +840,7 @@ PCODE CallCountingManager::OnCallCountThresholdReached(TransitionBlock *transiti
             }
 
             CallCountingInfo::Stage callCountingStage = callCountingInfo->GetStage();
-            if (callCountingStage >= CallCountingInfo::Stage::PendingCompletion)
+            if (callCountingStage >= CallCountingInfo::Stage::PendingReset)
             {
                 break;
             }
@@ -930,7 +973,11 @@ void CallCountingManager::CompleteCallCounting()
                     methodDesc->ResetCodeEntryPoint();
                 } while (false);
 
-                callCountingInfo->SetStage(CallCountingInfo::Stage::Complete);
+                // stub was about to be completed but we decided to re-use it
+                if (callCountingInfo->GetStage() != CallCountingInfo::Stage::PendingReset)
+                {
+                    callCountingInfo->SetStage(CallCountingInfo::Stage::Complete);
+                }
             }
             EX_CATCH
             {

src/coreclr/vm/callcounting.h

@@ -182,6 +182,9 @@ class CallCountingManager
             // Stub may be called, don't know if it's actually active (changes to code versions, etc.)
             StubMayBeActive,
 
+            // Stub is being considered to re-use
+            PendingReset,
+
             // Stub may be active, call counting complete, not yet promoted
             PendingCompletion,
 
@@ -193,7 +196,7 @@ class CallCountingManager
         };
 
     private:
-        const NativeCodeVersion m_codeVersion;
+        NativeCodeVersion m_codeVersion;
         const CallCountingStub *m_callCountingStub;
         CallCount m_remainingCallCount;
         Stage m_stage;
@@ -224,6 +227,7 @@ class CallCountingManager
     #ifndef DACCESS_COMPILE
     public:
         void SetStage(Stage stage);
+        void ReuseForNewVersion(NativeCodeVersion version);
     #endif
 
         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -355,6 +359,7 @@ class CallCountingManager
 #ifndef DACCESS_COMPILE
 public:
     void DisableCallCounting(NativeCodeVersion codeVersion);
+    void ReuseStubForNewVersion(NativeCodeVersion oldVersion, NativeCodeVersion newVersion);
 
 public:
     static bool SetCodeEntryPoint(

src/coreclr/vm/codeversion.cpp

@@ -333,6 +333,13 @@ NativeCodeVersion::OptimizationTier NativeCodeVersion::GetOptimizationTier() con
     }
 }
 
+bool NativeCodeVersion::IsUnoptimizedTier() const
+{
+    LIMITED_METHOD_DAC_CONTRACT;
+    OptimizationTier tier = GetOptimizationTier();
+    return tier == OptimizationTier0 || tier == OptimizationTier0Instrumented;
+}
+
 #ifndef DACCESS_COMPILE
 void NativeCodeVersion::SetOptimizationTier(OptimizationTier tier)
 {
@@ -808,7 +815,7 @@ bool ILCodeVersion::HasAnyOptimizedNativeCodeVersion(NativeCodeVersion tier0Nati
     _ASSERTE(!tier0NativeCodeVersion.IsNull());
     _ASSERTE(tier0NativeCodeVersion.GetILCodeVersion() == *this);
     _ASSERTE(tier0NativeCodeVersion.GetMethodDesc()->IsEligibleForTieredCompilation());
-    _ASSERTE(tier0NativeCodeVersion.GetOptimizationTier() == NativeCodeVersion::OptimizationTier0);
+    _ASSERTE(tier0NativeCodeVersion.IsUnoptimizedTier());
 
     NativeCodeVersionCollection nativeCodeVersions = GetNativeCodeVersions(tier0NativeCodeVersion.GetMethodDesc());
     for (auto itEnd = nativeCodeVersions.End(), it = nativeCodeVersions.Begin(); it != itEnd; ++it)
@@ -1710,7 +1717,7 @@ PCODE CodeVersionManager::PublishVersionableCodeIfNecessary(
                 _ASSERTE(!config->ShouldCountCalls() || pMethodDesc->IsEligibleForTieredCompilation());
                 _ASSERTE(
                     !config->ShouldCountCalls() ||
-                    activeVersion.GetOptimizationTier() == NativeCodeVersion::OptimizationTier0);
+                    activeVersion.IsUnoptimizedTier());
                 if (config->ShouldCountCalls()) // the generated code was at a tier that is call-counted
                 {
                     // This is the first call to a call-counted code version of the method

src/coreclr/vm/codeversion.h

@@ -74,12 +74,14 @@ class NativeCodeVersion
     enum OptimizationTier
     {
         OptimizationTier0,
+        OptimizationTier0Instrumented,
         OptimizationTier1,
         OptimizationTier1OSR,
         OptimizationTierOptimized, // may do less optimizations than tier 1
     };
 #ifdef FEATURE_TIERED_COMPILATION
     OptimizationTier GetOptimizationTier() const;
+    bool IsUnoptimizedTier() const;
 #ifndef DACCESS_COMPILE
     void SetOptimizationTier(OptimizationTier tier);
 #endif

src/coreclr/vm/eeconfig.cpp

@@ -740,6 +740,14 @@ HRESULT EEConfig::sync()
             }
         }
 
+        if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_TieredPGO) != 0)
+        {
+            DWORD divider = CLRConfig::GetConfigValue(CLRConfig::UNSUPPORTED_TC_DelayPgoDivider);
+            _ASSERT(divider != 0);
+            tieredCompilation_CallCountingDelayMs /= divider;
+            tieredCompilation_CallCountingDelayMs = max(1, tieredCompilation_CallCountingDelayMs);
+        }
+
         if (fTieredCompilation_CallCounting)
         {
             fTieredCompilation_UseCallCountingStubs =

src/coreclr/vm/interpreter.cpp

@@ -1802,7 +1802,7 @@ void Interpreter::JitMethodIfAppropriate(InterpreterMethodInfo* interpMethInfo,
             CodeVersionManager::LockHolder _lockHolder;
             NativeCodeVersion activeCodeVersion = md->GetCodeVersionManager()->GetActiveILCodeVersion(md).GetActiveNativeCodeVersion(md);
             ILCodeVersion ilCodeVersion = activeCodeVersion.GetILCodeVersion();
-            if (activeCodeVersion.GetOptimizationTier() == NativeCodeVersion::OptimizationTier0 &&
+            if (activeCodeVersion.IsUnoptimizedTier() &&
                 !ilCodeVersion.HasAnyOptimizedNativeCodeVersion(activeCodeVersion))
             {
                 tieredCompilationManager->AsyncPromoteToTier1(activeCodeVersion, &scheduleTieringBackgroundWork);

src/coreclr/vm/prestub.cpp

@@ -364,9 +364,9 @@ PCODE MethodDesc::PrepareILBasedCode(PrepareCodeConfig* pConfig)
         if (codeVersion.IsDefaultVersion())
         {
             pConfig->GetMethodDesc()->GetLoaderAllocator()->GetCallCountingManager()->DisableCallCounting(codeVersion);
-            _ASSERTE(codeVersion.GetOptimizationTier() != NativeCodeVersion::OptimizationTier0);
+            _ASSERTE(!codeVersion.IsUnoptimizedTier());
         }
-        else if (codeVersion.GetOptimizationTier() == NativeCodeVersion::OptimizationTier0)
+        else if (codeVersion.IsUnoptimizedTier())
         {
             codeVersion.SetOptimizationTier(NativeCodeVersion::OptimizationTierOptimized);
         }
@@ -465,7 +465,7 @@ PCODE MethodDesc::GetPrecompiledCode(PrepareCodeConfig* pConfig, bool shouldTier
 #ifdef FEATURE_TIERED_COMPILATION
                 if (shouldCountCalls)
                 {
-                    _ASSERTE(pConfig->GetCodeVersion().GetOptimizationTier() == NativeCodeVersion::OptimizationTier0);
+                    _ASSERTE(pConfig->GetCodeVersion().IsUnoptimizedTier());
                     pConfig->SetShouldCountCalls();
                 }
 #endif
@@ -1217,6 +1217,7 @@ PrepareCodeConfig::JitOptimizationTier PrepareCodeConfig::GetJitOptimizationTier
             switch (config->GetCodeVersion().GetOptimizationTier())
             {
                 case NativeCodeVersion::OptimizationTier0:
+                case NativeCodeVersion::OptimizationTier0Instrumented:
                     return JitOptimizationTier::QuickJitted;
 
                 case NativeCodeVersion::OptimizationTier1:
@@ -1299,6 +1300,7 @@ bool PrepareCodeConfig::FinalizeOptimizationTierForTier0LoadOrJit()
         NativeCodeVersion::OptimizationTier previousOptimizationTier = GetCodeVersion().GetOptimizationTier();
         _ASSERTE(
             previousOptimizationTier == NativeCodeVersion::OptimizationTier0 ||
+            previousOptimizationTier == NativeCodeVersion::OptimizationTier0Instrumented ||
             previousOptimizationTier == NativeCodeVersion::OptimizationTierOptimized);
     #endif // _DEBUG