Managed and unmanaged function pointers never

signature match.

src/coreclr/vm/siginfo.cpp

@@ -963,6 +963,30 @@ TypeHandle SigPointer::GetTypeHandleNT(Module* pModule,
 
 #endif // #ifndef DACCESS_COMPILE
 
+// Normalizing function pointer calling convention means
+// simply treating it as either "managed" or "unmanaged".
+static uint32_t NormalizeFnPtrCallingConvention(uint32_t callConv)
+{
+    LIMITED_METHOD_CONTRACT;
+
+    // Only have an unmanaged\managed status, and not the unmanaged CALLCONV_ value.
+    switch (callConv & IMAGE_CEE_CS_CALLCONV_MASK)
+    {
+    case IMAGE_CEE_CS_CALLCONV_C:
+    case IMAGE_CEE_CS_CALLCONV_STDCALL:
+    case IMAGE_CEE_CS_CALLCONV_THISCALL:
+    case IMAGE_CEE_CS_CALLCONV_FASTCALL:
+        // Strip the calling convention.
+        callConv &= ~IMAGE_CEE_CS_CALLCONV_MASK;
+        // Normalize to unmanaged.
+        callConv |= IMAGE_CEE_CS_CALLCONV_UNMANAGED;
+        break;
+    default:
+        break;
+    }
+
+    return callConv;
+}
 
 #ifdef _PREFAST_
 #pragma warning(push)
@@ -1739,7 +1763,7 @@ TypeHandle SigPointer::GetTypeHandleThrowing(
             TypeHandle *retAndArgTypes = (TypeHandle*) _alloca(cAllocaSize);
             bool fReturnTypeOrParameterNotLoaded = false;
 
-            for (unsigned i = 0; i <= cArgs; i++)
+            for (uint32_t i = 0; i <= cArgs; i++)
             {
                 // Lookup type handle.
                 retAndArgTypes[i] = psig.GetTypeHandleThrowing(pOrigModule,
@@ -1765,18 +1789,7 @@ TypeHandle SigPointer::GetTypeHandleThrowing(
                 break;
             }
 
-            // Only have an unmanaged\managed status, and not the unmanaged CALLCONV_ value.
-            switch (uCallConv & IMAGE_CEE_CS_CALLCONV_MASK)
-            {
-                case IMAGE_CEE_CS_CALLCONV_C:
-                case IMAGE_CEE_CS_CALLCONV_STDCALL:
-                case IMAGE_CEE_CS_CALLCONV_THISCALL:
-                case IMAGE_CEE_CS_CALLCONV_FASTCALL:
-                    // Strip the calling convention.
-                    uCallConv &= ~IMAGE_CEE_CS_CALLCONV_MASK;
-                    // Normalize to unmanaged.
-                    uCallConv |= IMAGE_CEE_CS_CALLCONV_UNMANAGED;
-            }
+            uCallConv = NormalizeFnPtrCallingConvention(uCallConv);
 
             // Find an existing function pointer or make a new one
             thRet = ClassLoader::LoadFnptrTypeThrowing((BYTE) uCallConv, cArgs, retAndArgTypes, fLoadTypes, level);
@@ -3929,15 +3942,15 @@ MetaSig::CompareElementType(
             IfFailThrow(CorSigUncompressElementType_EndPtr(pSig2, pEndSig2, &callingConvention2));
 
             // Calling conventions are generally treated as custom modifiers.
-            // When callers request calling conventions to be ignored, we also ignore
-            // calling conventions and this is okay. It is okay because calling conventions,
-            // when more than one is defined (for example, SuppressGCTransition), all
-            // become encoded as custom modifiers.
-            if (!state->IgnoreCustomModifiers
-                && callingConvention1 != callingConvention2)
-            {
+            // When callers request custom modifiers to be ignored, we also ignore
+            // specific unmanaged calling conventions and this is okay. It is okay
+            // because unmanaged calling conventions, when more than one is defined
+            // (for example, SuppressGCTransition), all become encoded as custom modifiers.
+            bool callConvMismatch = state->IgnoreCustomModifiers
+                ? NormalizeFnPtrCallingConvention(callingConvention1) != NormalizeFnPtrCallingConvention(callingConvention2)
+                : callingConvention1 != callingConvention2;
+            if (callConvMismatch)
                 return FALSE;
-            }
 
             DWORD argCnt1;
             IfFailThrow(CorSigUncompressData_EndPtr(pSig1, pEndSig1, &argCnt1));

src/tests/baseservices/compilerservices/UnsafeAccessors/UnsafeAccessorsTests.cs

@@ -26,6 +26,7 @@ class UserDataClass
         public const string MethodPointerName = nameof(_Pointer);
         public const string MethodCdeclCallConvBitName = nameof(_CdeclCallConvBit);
         public const string MethodStdcallCallConvBitName = nameof(_StdcallCallConvBit);
+        public const string MethodManagedCallConvBitName = nameof(_ManagedCallConvBit);
 
         private static string _F = PrivateStatic;
         private string _f;
@@ -51,10 +52,11 @@ private void _mvv() {}
         // Used to validate pointer values.
         private static string _Pointer(void* ptr) => "void*";
 
-        // Used to validate the embedded callconv bits (non-unmanaged bit) in
+        // Used to validate the embedded callconv bits in
         // ECMA-335 signatures for methods.
         private string _CdeclCallConvBit(delegate* unmanaged[Cdecl]<void> fptr) => nameof(CallConvCdecl);
         private string _StdcallCallConvBit(delegate* unmanaged[Stdcall]<void> fptr) => nameof(CallConvStdcall);
+        private string _ManagedCallConvBit(delegate* <void> fptr) => "Managed";
     }
 
     [StructLayout(LayoutKind.Sequential)]
@@ -330,9 +332,9 @@ public static void Verify_PreciseMatchCustomModifier()
 
     [Fact]
     [ActiveIssue("https://github.com/dotnet/runtime/issues/86040", TestRuntimes.Mono)]
-    public static void Verify_CallConvBitsAreTreatedAsCustomModifiersAndIgnored()
+    public static void Verify_UnmanagedCallConvBitAreTreatedAsCustomModifiersAndIgnored()
     {
-        Console.WriteLine($"Running {nameof(Verify_CallConvBitsAreTreatedAsCustomModifiersAndIgnored)}");
+        Console.WriteLine($"Running {nameof(Verify_UnmanagedCallConvBitAreTreatedAsCustomModifiersAndIgnored)}");
 
         var ud = CallPrivateConstructorClass();
         Assert.Equal(nameof(CallConvCdecl), CallCdeclMethod(ud, null));
@@ -350,6 +352,25 @@ public static void Verify_CallConvBitsAreTreatedAsCustomModifiersAndIgnored()
         extern static string CallStdcallMethod(UserDataClass d, delegate* unmanaged[Cdecl]<void> fptr);
     }
 
+    [Fact]
+    [ActiveIssue("https://github.com/dotnet/runtime/issues/86040", TestRuntimes.Mono)]
+    public static void Verify_ManagedUnmanagedFunctionPointersDontMatch()
+    {
+        Console.WriteLine($"Running {nameof(Verify_ManagedUnmanagedFunctionPointersDontMatch)}");
+
+        var ud = CallPrivateConstructorClass();
+        Assert.Throws<MissingMethodException>(() => CallCdeclMethod(ud, null));
+        Assert.Throws<MissingMethodException>(() => CallManagedMethod(ud, null));
+
+        // Managed calling conventions don't match on unmanaged function pointers
+        [UnsafeAccessor(UnsafeAccessorKind.Method, Name=UserDataClass.MethodCdeclCallConvBitName)]
+        extern static string CallCdeclMethod(UserDataClass d, delegate* <void> fptr);
+
+        // Unmanaged calling conventions don't match on managed function pointers
+        [UnsafeAccessor(UnsafeAccessorKind.Method, Name=UserDataClass.MethodManagedCallConvBitName)]
+        extern static string CallManagedMethod(UserDataClass d, delegate* unmanaged[Cdecl]<void> fptr);
+    }
+
     [Fact]
     [ActiveIssue("https://github.com/dotnet/runtime/issues/86040", TestRuntimes.Mono)]
     public static void Verify_InvalidTargetUnsafeAccessor()