Implement support in crossgen2 and NativeAOT

src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs

@@ -3747,7 +3747,7 @@ private uint getJitFlags(ref CORJIT_FLAGS flags, uint sizeInBytes)
 
 #if READYTORUN
                 // TODO: enable this check in full AOT
-                if (Marshaller.IsMarshallingRequired(this.MethodBeingCompiled.Signature, Array.Empty<ParameterMetadata>())) // Only blittable arguments
+                if (Marshaller.IsMarshallingRequired(this.MethodBeingCompiled.Signature, Array.Empty<ParameterMetadata>(), ((MetadataType)this.MethodBeingCompiled.OwningType).Module)) // Only blittable arguments
                 {
                     ThrowHelper.ThrowInvalidProgramException(ExceptionStringID.InvalidProgramNonBlittableTypes, this.MethodBeingCompiled);
                 }

src/coreclr/tools/Common/TypeSystem/IL/Stubs/CalliMarshallingMethodThunk.cs

@@ -21,11 +21,13 @@ public partial class CalliMarshallingMethodThunk : ILStubMethod
         private MethodSignature _signature;
 
         public CalliMarshallingMethodThunk(MethodSignature targetSignature, TypeDesc owningType,
-                InteropStateManager interopStateManager)
+                InteropStateManager interopStateManager,
+                bool useRuntimeMarshalling)
         {
             _targetSignature = targetSignature;
             _owningType = owningType;
             _interopStateManager = interopStateManager;
+            UseRuntimeMarshalling = useRuntimeMarshalling;
         }
 
         public MethodSignature TargetSignature
@@ -79,6 +81,8 @@ public override string Name
             }
         }
 
+        public bool UseRuntimeMarshalling { get; }
+
         public override PInvokeMetadata GetPInvokeMethodMetadata()
         {
             // Return PInvokeAttributes.PreserveSig to circumvent marshalling required checks

src/coreclr/tools/Common/TypeSystem/IL/Stubs/PInvokeILEmitter.cs

@@ -55,17 +55,21 @@ private static Marshaller[] InitializeMarshallers(MethodDesc targetMethod, Inter
         {
             MarshalDirection direction = MarshalDirection.Forward;
             MethodSignature methodSig;
+            bool runtimeMarshallingEnabled;
             switch (targetMethod)
             {
                 case DelegateMarshallingMethodThunk delegateMethod:
                     methodSig = delegateMethod.DelegateSignature;
                     direction = delegateMethod.Direction;
+                    runtimeMarshallingEnabled = MarshalHelpers.IsRuntimeMarshallingEnabled(delegateMethod.DelegateType.Module);
                     break;
                 case CalliMarshallingMethodThunk calliMethod:
                     methodSig = calliMethod.TargetSignature;
+                    runtimeMarshallingEnabled = calliMethod.UseRuntimeMarshalling;
                     break;
                 default:
                     methodSig = targetMethod.Signature;
+                    runtimeMarshallingEnabled = MarshalHelpers.IsRuntimeMarshallingEnabled(((EcmaMethod)targetMethod).Module);
                     break;
             }
             int indexOffset = 0;
@@ -114,20 +118,36 @@ private static Marshaller[] InitializeMarshallers(MethodDesc targetMethod, Inter
                     parameterType = methodSig[i - 1];
                 }
 
-                marshallers[i] = Marshaller.CreateMarshaller(parameterType,
-                                                    parameterIndex,
-                                                    methodSig.GetEmbeddedSignatureData(),
-                                                    MarshallerType.Argument,
-                                                    parameterMetadata.MarshalAsDescriptor,
-                                                    direction,
-                                                    marshallers,
-                                                    interopStateManager,
-                                                    indexOffset + parameterMetadata.Index,
-                                                    flags,
-                                                    parameterMetadata.In,
-                                                    isHRSwappedRetVal ? true : parameterMetadata.Out,
-                                                    isHRSwappedRetVal ? false : parameterMetadata.Return
-                                                    );
+                if (runtimeMarshallingEnabled)
+                {
+                    marshallers[i] = Marshaller.CreateMarshaller(parameterType,
+                                                        parameterIndex,
+                                                        methodSig.GetEmbeddedSignatureData(),
+                                                        MarshallerType.Argument,
+                                                        parameterMetadata.MarshalAsDescriptor,
+                                                        direction,
+                                                        marshallers,
+                                                        interopStateManager,
+                                                        indexOffset + parameterMetadata.Index,
+                                                        flags,
+                                                        parameterMetadata.In,
+                                                        isHRSwappedRetVal ? true : parameterMetadata.Out,
+                                                        isHRSwappedRetVal ? false : parameterMetadata.Return
+                                                        );
+                }
+                else
+                {
+                    marshallers[i] = Marshaller.CreateDisabledRuntimeMarshallingMarshaller(
+                        parameterType,
+                        parameterIndex,
+                        MarshallerType.Argument,
+                        direction,
+                        marshallers,
+                        interopStateManager,
+                        indexOffset + parameterMetadata.Index,
+                        flags,
+                        parameterMetadata.Return);
+                }
             }
 
             return marshallers;

src/coreclr/tools/Common/TypeSystem/Interop/IL/MarshalHelpers.Aot.cs

@@ -126,7 +126,7 @@ public static MethodIL EmitExceptionBody(string message, MethodDesc method)
             TypeSystemContext context = method.Context;
             MethodSignature ctorSignature = new MethodSignature(0, 0, context.GetWellKnownType(WellKnownType.Void),
                 new TypeDesc[] { context.GetWellKnownType(WellKnownType.String) });
-            MethodDesc exceptionCtor = method.Context.GetWellKnownType(WellKnownType.Exception).GetKnownMethod(".ctor", ctorSignature);
+            MethodDesc exceptionCtor = InteropTypes.GetMarshalDirectiveException(context).GetKnownMethod(".ctor", ctorSignature);
 
             ILCodeStream codeStream = emitter.NewCodeStream();
             codeStream.Emit(ILOpcode.ldstr, emitter.NewToken(message));

src/coreclr/tools/Common/TypeSystem/Interop/IL/MarshalHelpers.cs

@@ -5,6 +5,7 @@
 using Internal.IL;
 using Debug = System.Diagnostics.Debug;
 using Internal.IL.Stubs;
+using Internal.TypeSystem.Ecma;
 
 namespace Internal.TypeSystem.Interop
 {
@@ -844,6 +845,24 @@ private static MarshallerKind GetArrayElementMarshallerKind(
             }
         }
 
+        internal static MarshallerKind GetRuntimeDisabledMarshallingMarshallerKind(
+            TypeDesc type)
+        {
+            if (type.IsPrimitive && type.Category != TypeFlags.Void)
+            {
+                return MarshallerKind.BlittableValue;
+            }
+            else if (type.IsPointer || type.IsFunctionPointer)
+            {
+                return MarshallerKind.BlittableValue;
+            }
+            else if (type.IsValueType && !((DefType)type).ContainsGCPointers)
+            {
+                return MarshallerKind.BlittableValue;
+            }
+            return MarshallerKind.Invalid;
+        }
+
         internal static bool ShouldCheckForPendingException(TargetDetails target, PInvokeMetadata metadata)
         {
             if (!target.IsOSX)
@@ -861,5 +880,10 @@ internal static bool ShouldCheckForPendingException(TargetDetails target, PInvok
             return metadata.Module.Equals(ObjectiveCLibrary)
                 && metadata.Name.StartsWith(ObjectiveCMsgSend);
         }
+
+        public static bool IsRuntimeMarshallingEnabled(ModuleDesc module)
+        {
+            return module.Assembly is not EcmaAssembly assembly || assembly.HasAssemblyCustomAttribute("System.Runtime.CompilerServices", "DisableRuntimeMarshallingAttribute");
+        }
     }
 }

src/coreclr/tools/Common/TypeSystem/Interop/IL/Marshaller.cs

@@ -382,6 +382,58 @@ public static Marshaller CreateMarshaller(TypeDesc parameterType,
 
             return marshaller;
         }
+
+        /// <summary>
+        /// Create a marshaller
+        /// </summary>
+        /// <param name="parameterType">type of the parameter to marshal</param>
+        /// <returns>The created Marshaller</returns>
+        public static Marshaller CreateDisabledRuntimeMarshallingMarshaller(TypeDesc parameterType,
+            int? parameterIndex,
+            MarshallerType marshallerType,
+            MarshalDirection direction,
+            Marshaller[] marshallers,
+#if !READYTORUN
+            InteropStateManager interopStateManager,
+#endif
+            int index,
+            PInvokeFlags flags,
+            bool isReturn)
+        {
+            bool isAnsi = flags.CharSet switch
+            {
+                CharSet.Ansi => true,
+                CharSet.Unicode => false,
+                CharSet.Auto => !parameterType.Context.Target.IsWindows,
+                _ => true
+            };
+
+            MarshallerKind marshallerKind = MarshalHelpers.GetRuntimeDisabledMarshallingMarshallerKind(parameterType);
+
+            TypeSystemContext context = parameterType.Context;
+            // Create the marshaller based on MarshallerKind
+            Marshaller marshaller = CreateMarshaller(marshallerKind);
+            marshaller.Context = context;
+#if !READYTORUN
+            marshaller.InteropStateManager = interopStateManager;
+#endif
+            marshaller.MarshallerKind = marshallerKind;
+            marshaller.MarshallerType = marshallerType;
+            marshaller.ElementMarshallerKind = MarshallerKind.Unknown;
+            marshaller.ManagedParameterType = parameterType;
+            marshaller.ManagedType = parameterType.IsByRef ? parameterType.GetParameterType() : parameterType;
+            marshaller.Return = isReturn;
+            marshaller.IsManagedByRef = parameterType.IsByRef;
+            marshaller.IsNativeByRef = marshaller.IsManagedByRef;
+            marshaller.MarshalDirection = direction;
+            marshaller.MarshalAsDescriptor = null;
+            marshaller.Marshallers = marshallers;
+            marshaller.Index = index;
+            marshaller.PInvokeFlags = flags;
+            marshaller.In = true;
+            marshaller.Out = marshaller.IsManagedByRef;
+            return marshaller;
+        }
         #endregion
 
 

src/coreclr/tools/Common/TypeSystem/Interop/InteropStateManager.cs

@@ -186,9 +186,9 @@ public FieldDesc GetPInvokeLazyFixupField(MethodDesc method)
             return _pInvokeLazyFixupFieldHashtable.GetOrCreateValue(method);
         }
 
-        public MethodDesc GetPInvokeCalliStub(MethodSignature signature)
+        public MethodDesc GetPInvokeCalliStub(MethodSignature signature, bool useRuntimeMarshalling)
         {
-            return _pInvokeCalliHashtable.GetOrCreateValue(signature);
+            return _pInvokeCalliHashtable.GetOrCreateValue((signature, useRuntimeMarshalling));
         }
 
         private class NativeStructTypeHashtable : LockFreeReaderHashtable<MetadataType, NativeStructType>
@@ -478,12 +478,12 @@ public PInvokeLazyFixupFieldHashtable(DefType owningType)
             }
         }
 
-        private class PInvokeCalliHashtable : LockFreeReaderHashtable<MethodSignature, CalliMarshallingMethodThunk>
+        private class PInvokeCalliHashtable : LockFreeReaderHashtable<(MethodSignature Signature, bool UseRuntimeMarshalling), CalliMarshallingMethodThunk>
         {
             private readonly InteropStateManager _interopStateManager;
             private readonly TypeDesc _owningType;
 
-            protected override int GetKeyHashCode(MethodSignature key)
+            protected override int GetKeyHashCode((MethodSignature Signature, bool UseRuntimeMarshalling) key)
             {
                 return key.GetHashCode();
             }
@@ -493,7 +493,7 @@ protected override int GetValueHashCode(CalliMarshallingMethodThunk value)
                 return value.TargetSignature.GetHashCode();
             }
 
-            protected override bool CompareKeyToValue(MethodSignature key, CalliMarshallingMethodThunk value)
+            protected override bool CompareKeyToValue((MethodSignature Signature, bool UseRuntimeMarshalling) key, CalliMarshallingMethodThunk value)
             {
                 return key.Equals(value.TargetSignature);
             }
@@ -503,9 +503,9 @@ protected override bool CompareValueToValue(CalliMarshallingMethodThunk value1,
                 return value1.TargetSignature.Equals(value2.TargetSignature);
             }
 
-            protected override CalliMarshallingMethodThunk CreateValueFromKey(MethodSignature key)
+            protected override CalliMarshallingMethodThunk CreateValueFromKey((MethodSignature Signature, bool UseRuntimeMarshalling) key)
             {
-                return new CalliMarshallingMethodThunk(key, _owningType, _interopStateManager);
+                return new CalliMarshallingMethodThunk(key.Signature, _owningType, _interopStateManager, key.UseRuntimeMarshalling);
             }
 
             public PInvokeCalliHashtable(InteropStateManager interopStateManager, TypeDesc owningType)

src/coreclr/tools/Common/TypeSystem/Interop/InteropTypes.cs

@@ -54,6 +54,11 @@ public static MetadataType GetNativeFunctionPointerWrapper(TypeSystemContext con
             return context.SystemModule.GetKnownType("System.Runtime.InteropServices", "NativeFunctionPointerWrapper");
         }
 
+        public static MetadataType GetMarshalDirectiveException(TypeSystemContext context)
+        {
+            return context.SystemModule.GetKnownType("System.Runtime.InteropServices", "MarshalDirectiveException");
+        }
+
         public static MetadataType GetVariant(TypeSystemContext context)
         {
             return context.SystemModule.GetKnownType("System.Runtime.InteropServices", "Variant");

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/UsageBasedInteropStubManager.cs

@@ -25,7 +25,7 @@ public UsageBasedInteropStubManager(InteropStateManager interopStateManager, PIn
 
         public override void AddDependeciesDueToPInvoke(ref DependencyList dependencies, NodeFactory factory, MethodDesc method)
         {
-            if (method.IsPInvoke)
+            if (method.IsPInvoke && method.OwningType is MetadataType type && MarshalHelpers.IsRuntimeMarshallingEnabled(type.Module))
             {
                 dependencies = dependencies ?? new DependencyList();
 

src/coreclr/tools/aot/ILCompiler.Compiler/IL/Stubs/PInvokeILProvider.cs

@@ -30,9 +30,9 @@ public override MethodIL GetMethodIL(MethodDesc method)
             return PInvokeILEmitter.EmitIL(method, _pInvokeILEmitterConfiguration, _interopStateManager);
         }
 
-        public MethodDesc GetCalliStub(MethodSignature signature)
+        public MethodDesc GetCalliStub(MethodSignature signature, bool useRuntimeMarshalling)
         {
-            return _interopStateManager.GetPInvokeCalliStub(signature);
+            return _interopStateManager.GetPInvokeCalliStub(signature, useRuntimeMarshalling);
         }
 
         public string GetDirectCallExternName(MethodDesc method)

src/coreclr/tools/aot/ILCompiler.ReadyToRun/IL/Stubs/PInvokeILEmitter.cs

@@ -12,6 +12,7 @@
 using Internal.JitInterface;
 
 using Debug = System.Diagnostics.Debug;
+using Internal.TypeSystem.Ecma;
 
 namespace Internal.IL.Stubs
 {
@@ -46,6 +47,15 @@ private void EmitPInvokeCall(PInvokeILCodeStreams ilCodeStreams)
             // if the SetLastError flag is set in DllImport, clear the error code before doing P/Invoke
             if (_importMetadata.Flags.SetLastError)
             {
+                if (_targetMethod is EcmaMethod ecmaMethod
+                    && ecmaMethod.Module.Assembly is EcmaAssembly assembly
+                    && assembly.HasAssemblyCustomAttribute("System.Runtime.CompilerServices", "DisableRuntimeMarshallingAttribute"))
+                {
+                    // When runtime marshalling is disabled, we don't support generating the stub IL
+                    // in Ready-to-Run so we can correctly throw an exception at runtime when the user tries to
+                    // use SetLastError=true when marshalling is disabled.
+                    throw new NotSupportedException();
+                }
                 callsiteSetupCodeStream.Emit(ILOpcode.call, emitter.NewToken(
                             stubHelpersType.GetKnownMethod("ClearLastError", null)));
             }