Add support for storing method handle histograms in profiles

Allow method handle histograms in .mibc files and in the PGO text
format.

Contributes to #44610.

src/coreclr/inc/corjit.h

@@ -368,6 +368,7 @@ class ICorJitInfo : public ICorDynamicInfo
         FourByte = 1,
         EightByte = 2,
         TypeHandle = 3,
+        MethodHandle = 4,
 
         // Mask of all schema data types
         MarshalMask = 0xF,
@@ -385,9 +386,10 @@ class ICorJitInfo : public ICorDynamicInfo
         Done = None, // All instrumentation schemas must end with a record which is "Done"
         BasicBlockIntCount = (DescriptorMin * 1) | FourByte, // basic block counter using unsigned 4 byte int
         BasicBlockLongCount = (DescriptorMin * 1) | EightByte, // basic block counter using unsigned 8 byte int
-        TypeHandleHistogramIntCount = (DescriptorMin * 2) | FourByte | AlignPointer, // 4 byte counter that is part of a type histogram. Aligned to match ClassProfile32's alignment.
-        TypeHandleHistogramLongCount = (DescriptorMin * 2) | EightByte, // 8 byte counter that is part of a type histogram
-        TypeHandleHistogramTypeHandle = (DescriptorMin * 3) | TypeHandle, // TypeHandle that is part of a type histogram
+        HandleHistogramIntCount = (DescriptorMin * 2) | FourByte | AlignPointer, // 4 byte counter that is part of a type histogram. Aligned to match ClassProfile32's alignment.
+        HandleHistogramLongCount = (DescriptorMin * 2) | EightByte, // 8 byte counter that is part of a type histogram
+        HandleHistogramTypes = (DescriptorMin * 3) | TypeHandle, // Histogram of type handles
+        HandleHistogramMethods = (DescriptorMin * 3) | MethodHandle, // Histogram of method handles
         Version = (DescriptorMin * 4) | None, // Version is encoded in the Other field of the schema
         NumRuns = (DescriptorMin * 5) | None, // Number of runs is encoded in the Other field of the schema
         EdgeIntCount = (DescriptorMin * 6) | FourByte, // edge counter using unsigned 4 byte int
@@ -416,12 +418,12 @@ class ICorJitInfo : public ICorDynamicInfo
     };
 
 #define DEFAULT_UNKNOWN_TYPEHANDLE 1
-#define UNKNOWN_TYPEHANDLE_MIN 1
-#define UNKNOWN_TYPEHANDLE_MAX 33
+#define UNKNOWN_HANDLE_MIN 1
+#define UNKNOWN_HANDLE_MAX 33
 
-    static inline bool IsUnknownTypeHandle(intptr_t typeHandle)
+    static inline bool IsUnknownHandle(intptr_t typeHandle)
     {
-        return ((typeHandle >= UNKNOWN_TYPEHANDLE_MIN) && (typeHandle <= UNKNOWN_TYPEHANDLE_MAX));
+        return ((typeHandle >= UNKNOWN_HANDLE_MIN) && (typeHandle <= UNKNOWN_HANDLE_MAX));
     }
 
     // get profile information to be used for optimizing a current method.  The format

src/coreclr/inc/pgo_formatprocessing.h

@@ -8,15 +8,7 @@
 
 #ifdef FEATURE_PGO
 
-inline bool AddTypeHandleToUnknownTypeHandleMask(INT_PTR typeHandle, uint32_t *unknownTypeHandleMask)
-{
-    uint32_t bitMask = (uint32_t)(1 << (typeHandle - UNKNOWN_TYPEHANDLE_MIN));
-    bool result = (bitMask & *unknownTypeHandleMask) == 0;
-    *unknownTypeHandleMask |= bitMask;
-    return result;
-}
-
-inline INT_PTR HashToPgoUnknownTypeHandle(uint32_t hash)
+inline INT_PTR HashToPgoUnknownHandle(uint32_t hash)
 {
     // Map from a 32bit hash to the 32 different unknown type handle values
     return (hash & 0x1F) + 1;
@@ -53,6 +45,7 @@ inline uint32_t InstrumentationKindToSize(ICorJitInfo::PgoInstrumentationKind ki
         case ICorJitInfo::PgoInstrumentationKind::EightByte:
             return 8;
         case ICorJitInfo::PgoInstrumentationKind::TypeHandle:
+        case ICorJitInfo::PgoInstrumentationKind::MethodHandle:
             return TARGET_POINTER_SIZE;
         default:
             _ASSERTE(FALSE);
@@ -242,6 +235,7 @@ bool ReadInstrumentationData(const uint8_t *pByte, size_t cbDataMax, SchemaAndDa
     bool done = false;
     int64_t lastDataValue = 0;
     int64_t lastTypeDataValue = 0;
+    int64_t lastMethodDataValue = 0;
     int32_t dataCountToRead = 0;
 
     ReadCompressedInts(pByte, cbDataMax, [&](int64_t curValue)
@@ -267,8 +261,16 @@ bool ReadInstrumentationData(const uint8_t *pByte, size_t cbDataMax, SchemaAndDa
                     return false;
                 }
                 break;
+            case ICorJitInfo::PgoInstrumentationKind::MethodHandle:
+                lastMethodDataValue += curValue;
+
+                if (!handler(schemaHandler.GetSchema(), lastMethodDataValue, schemaHandler.GetSchema().Count - dataCountToRead))
+                {
+                    return false;
+                }
+                break;
             default:
-                assert(false);
+                assert(!"Unexpected PGO instrumentation data type");
                 break;
             }
             dataCountToRead--;

src/coreclr/jit/compiler.h

@@ -6264,6 +6264,7 @@ class Compiler
     UINT32                                 fgPgoBlockCounts;
     UINT32                                 fgPgoEdgeCounts;
     UINT32                                 fgPgoClassProfiles;
+    UINT32                                 fgPgoMethodProfiles;
     unsigned                               fgPgoInlineePgo;
     unsigned                               fgPgoInlineeNoPgo;
     unsigned                               fgPgoInlineeNoPgoSingleBlock;

src/coreclr/jit/fgbasic.cpp

@@ -180,6 +180,7 @@ void Compiler::fgInit()
     fgPgoBlockCounts             = 0;
     fgPgoEdgeCounts              = 0;
     fgPgoClassProfiles           = 0;
+    fgPgoMethodProfiles          = 0;
     fgPgoInlineePgo              = 0;
     fgPgoInlineeNoPgo            = 0;
     fgPgoInlineeNoPgoSingleBlock = 0;

src/coreclr/jit/fgprofile.cpp

@@ -1496,15 +1496,15 @@ class BuildClassProbeSchemaGen
         }
 
         schemaElem.InstrumentationKind = JitConfig.JitCollect64BitCounts()
-                                             ? ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount
-                                             : ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount;
+                                             ? ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount
+                                             : ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount;
         schemaElem.ILOffset = (int32_t)call->gtClassProfileCandidateInfo->ilOffset;
         schemaElem.Offset   = 0;
 
         m_schema.push_back(schemaElem);
 
         // Re-using ILOffset and Other fields from schema item for TypeHandleHistogramCount
-        schemaElem.InstrumentationKind = ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramTypeHandle;
+        schemaElem.InstrumentationKind = ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes;
         schemaElem.Count               = ICorJitInfo::ClassProfile32::SIZE;
         m_schema.push_back(schemaElem);
 
@@ -1551,9 +1551,9 @@ class ClassProbeInserter
         //
         assert(m_schema[*m_currentSchemaIndex].ILOffset == (int32_t)call->gtClassProfileCandidateInfo->ilOffset);
         bool is32 = m_schema[*m_currentSchemaIndex].InstrumentationKind ==
-                    ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount;
+                    ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount;
         bool is64 = m_schema[*m_currentSchemaIndex].InstrumentationKind ==
-                    ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount;
+                    ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount;
         assert(is32 || is64);
 
         // Figure out where the table is located.
@@ -2050,12 +2050,32 @@ PhaseStatus Compiler::fgIncorporateProfileData()
                 fgPgoEdgeCounts++;
                 break;
 
-            case ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount:
-            case ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount:
             case ICorJitInfo::PgoInstrumentationKind::GetLikelyClass:
                 fgPgoClassProfiles++;
                 break;
 
+            case ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount:
+            case ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount:
+                if (iSchema + 1 < fgPgoSchemaCount)
+                {
+                    if (fgPgoSchema[iSchema + 1].InstrumentationKind ==
+                        ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes)
+                    {
+                        fgPgoClassProfiles++;
+                        iSchema++;
+                        break;
+                    }
+                    if (fgPgoSchema[iSchema + 1].InstrumentationKind ==
+                        ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods)
+                    {
+                        fgPgoMethodProfiles++;
+                        iSchema++;
+                        break;
+                    }
+                }
+
+                __fallthrough;
+
             default:
                 JITDUMP("Unknown PGO record type 0x%x in schema entry %u (offset 0x%x count 0x%x other 0x%x)\n",
                         fgPgoSchema[iSchema].InstrumentationKind, iSchema, fgPgoSchema[iSchema].ILOffset,
@@ -2070,8 +2090,9 @@ PhaseStatus Compiler::fgIncorporateProfileData()
         fgNumProfileRuns = 1;
     }
 
-    JITDUMP("Profile summary: %d runs, %d block probes, %d edge probes, %d class profiles, %d other records\n",
-            fgNumProfileRuns, fgPgoBlockCounts, fgPgoEdgeCounts, fgPgoClassProfiles, otherRecords);
+    JITDUMP("Profile summary: %d runs, %d block probes, %d edge probes, %d class profiles, %d method profiles, %d "
+            "other records\n",
+            fgNumProfileRuns, fgPgoBlockCounts, fgPgoEdgeCounts, fgPgoClassProfiles, fgPgoMethodProfiles, otherRecords);
 
     const bool haveBlockCounts = fgPgoBlockCounts > 0;
     const bool haveEdgeCounts  = fgPgoEdgeCounts > 0;

src/coreclr/jit/likelyclass.cpp

@@ -157,7 +157,7 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord*
             (schema[i].Count == 1))
         {
             INT_PTR result = *(INT_PTR*)(pInstrumentationData + schema[i].Offset);
-            if (ICorJitInfo::IsUnknownTypeHandle(result))
+            if (ICorJitInfo::IsUnknownHandle(result))
             {
                 return 0;
             }
@@ -168,11 +168,11 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord*
         }
 
         const bool isHistogramCount =
-            (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount) ||
-            (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount);
+            (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount) ||
+            (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount);
 
         if (isHistogramCount && (schema[i].Count == 1) && ((i + 1) < countSchemaItems) &&
-            (schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramTypeHandle))
+            (schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes))
         {
             // Form a histogram
             //
@@ -191,7 +191,7 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord*
                 {
                     LikelyClassHistogramEntry const hist0 = h.HistogramEntryAt(0);
                     // Fast path for monomorphic cases
-                    if (ICorJitInfo::IsUnknownTypeHandle(hist0.m_mt))
+                    if (ICorJitInfo::IsUnknownHandle(hist0.m_mt))
                     {
                         return 0;
                     }
@@ -205,12 +205,12 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord*
                     LikelyClassHistogramEntry const hist0 = h.HistogramEntryAt(0);
                     LikelyClassHistogramEntry const hist1 = h.HistogramEntryAt(1);
                     // Fast path for two classes
-                    if ((hist0.m_count >= hist1.m_count) && !ICorJitInfo::IsUnknownTypeHandle(hist0.m_mt))
+                    if ((hist0.m_count >= hist1.m_count) && !ICorJitInfo::IsUnknownHandle(hist0.m_mt))
                     {
                         pLikelyClasses[0].likelihood = (100 * hist0.m_count) / h.m_totalCount;
                         pLikelyClasses[0].clsHandle  = (CORINFO_CLASS_HANDLE)hist0.m_mt;
 
-                        if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownTypeHandle(hist1.m_mt))
+                        if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownHandle(hist1.m_mt))
                         {
                             pLikelyClasses[1].likelihood = (100 * hist1.m_count) / h.m_totalCount;
                             pLikelyClasses[1].clsHandle  = (CORINFO_CLASS_HANDLE)hist1.m_mt;
@@ -219,12 +219,12 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord*
                         return 1;
                     }
 
-                    if (!ICorJitInfo::IsUnknownTypeHandle(hist1.m_mt))
+                    if (!ICorJitInfo::IsUnknownHandle(hist1.m_mt))
                     {
                         pLikelyClasses[0].likelihood = (100 * hist1.m_count) / h.m_totalCount;
                         pLikelyClasses[0].clsHandle  = (CORINFO_CLASS_HANDLE)hist1.m_mt;
 
-                        if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownTypeHandle(hist0.m_mt))
+                        if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownHandle(hist0.m_mt))
                         {
                             pLikelyClasses[1].likelihood = (100 * hist0.m_count) / h.m_totalCount;
                             pLikelyClasses[1].clsHandle  = (CORINFO_CLASS_HANDLE)hist0.m_mt;
@@ -244,7 +244,7 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord*
                     for (unsigned m = 0; m < h.countHistogramElements; m++)
                     {
                         LikelyClassHistogramEntry const hist = h.HistogramEntryAt(m);
-                        if (!ICorJitInfo::IsUnknownTypeHandle(hist.m_mt))
+                        if (!ICorJitInfo::IsUnknownHandle(hist.m_mt))
                         {
                             sortedEntries[knownHandles++] = hist;
                         }
@@ -311,7 +311,7 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch
             (schema[i].Count == 1))
         {
             INT_PTR result = *(INT_PTR*)(pInstrumentationData + schema[i].Offset);
-            if (ICorJitInfo::IsUnknownTypeHandle(result))
+            if (ICorJitInfo::IsUnknownHandle(result))
             {
                 return NO_CLASS_HANDLE;
             }
@@ -322,11 +322,11 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch
         }
 
         bool isHistogramCount =
-            (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount) ||
-            (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount);
+            (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount) ||
+            (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount);
 
         if (isHistogramCount && (schema[i].Count == 1) && ((i + 1) < countSchemaItems) &&
-            (schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramTypeHandle))
+            (schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes))
         {
             // Form a histogram
             //
@@ -342,7 +342,7 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch
             unsigned                  randomEntryIndex = random->Next(0, h.countHistogramElements);
             LikelyClassHistogramEntry randomEntry      = h.HistogramEntryAt(randomEntryIndex);
 
-            if (ICorJitInfo::IsUnknownTypeHandle(randomEntry.m_mt))
+            if (ICorJitInfo::IsUnknownHandle(randomEntry.m_mt))
             {
                 return NO_CLASS_HANDLE;
             }

src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs

@@ -195,8 +195,7 @@ public static IEnumerable<PgoSchemaElem> ConvertTypeHandleHistogramsToCompactTyp
             bool hasTypeHistogram = false;
             foreach (var elem in pgoData)
             {
-                if (elem.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramIntCount ||
-                    elem.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramLongCount)
+                if (elem.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes)
                 {
                     // found histogram
                     hasTypeHistogram = true;
@@ -218,10 +217,12 @@ public static IEnumerable<PgoSchemaElem> ConvertTypeHandleHistogramsToCompactTyp
 
                 ComputeJitPgoInstrumentationSchema(LocalObjectToHandle, pgoData, out var nativeSchema, out var instrumentationData);
 
-                for (int i = 0; i < (pgoData.Length); i++)
+                for (int i = 0; i < pgoData.Length; i++)
                 {
-                    if (pgoData[i].InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramIntCount ||
-                        pgoData[i].InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramLongCount)
+                    if ((i + 1 < pgoData.Length) &&
+                        (pgoData[i].InstrumentationKind == PgoInstrumentationKind.HandleHistogramIntCount ||
+                         pgoData[i].InstrumentationKind == PgoInstrumentationKind.HandleHistogramLongCount) &&
+                        (pgoData[i + 1].InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes))
                     {
                         PgoSchemaElem? newElem = ComputeLikelyClass(i, handleToObject, nativeSchema, instrumentationData, compilationModuleGroup);
                         if (newElem.HasValue)
@@ -3863,6 +3864,10 @@ public static void ComputeJitPgoInstrumentationSchema(Func<object, IntPtr> objec
                             {
                                 ptrVal = (IntPtr)objectToHandle(typeVal.AsType);
                             }
+                            else if (typeVal.AsMethod != null)
+                            {
+                                ptrVal = (IntPtr)objectToHandle(typeVal.AsMethod);
+                            }
                             else
                             {
                                 // The "Unknown types are the values from 1-33