Disable Int128 use in by value ABI scenarios, and fix field layout behavior

src/coreclr/dlls/mscorrc/mscorrc.rc

@@ -271,6 +271,7 @@ BEGIN
     IDS_EE_BADMARSHAL_ABSTRACTRETCRITICALHANDLE "Returned CriticalHandles cannot be abstract."
     IDS_EE_BADMARSHAL_CUSTOMMARSHALER       "Custom marshalers are only allowed on classes, strings, arrays, and boxed value types."
     IDS_EE_BADMARSHAL_GENERICS_RESTRICTION  "Non-blittable generic types cannot be marshaled."
+    IDS_EE_BADMARSHAL_INT128_RESTRICTION    "System.Int128 and System.UInt128 cannot be passed by value to unmanaged."
     IDS_EE_BADMARSHAL_AUTOLAYOUT            "Structures marked with [StructLayout(LayoutKind.Auto)] cannot be marshaled."
     IDS_EE_BADMARSHAL_STRING_OUT            "Cannot marshal a string by-value with the [Out] attribute."
     IDS_EE_BADMARSHAL_MARSHAL_DISABLED      "Cannot marshal managed types when the runtime marshalling system is disabled."

src/coreclr/dlls/mscorrc/resource.h

@@ -283,6 +283,7 @@
 #define IDS_EE_BADMARSHAL_ABSTRACTOUTCRITICALHANDLE 0x1a63
 #define IDS_EE_BADMARSHAL_RETURNCHCOMTONATIVE       0x1a64
 #define IDS_EE_BADMARSHAL_CRITICALHANDLE            0x1a65
+#define IDS_EE_BADMARSHAL_INT128_RESTRICTION        0x1a66
 
 #define IDS_EE_BADMARSHAL_ABSTRACTRETCRITICALHANDLE 0x1a6a
 #define IDS_EE_CH_IN_VARIANT_NOT_SUPPORTED          0x1a6b

src/coreclr/tools/Common/Compiler/Int128FieldLayoutAlgorithm.cs

@@ -28,8 +28,10 @@ public override ComputedInstanceFieldLayout ComputeInstanceLayout(DefType defTyp
 
             ComputedInstanceFieldLayout layoutFromMetadata = _fallbackAlgorithm.ComputeInstanceLayout(defType, layoutKind);
 
-            if (defType.Context.Target.IsWindows || (defType.Context.Target.PointerSize == 4))
+            // 32bit platforms use standard metadata layout engine
+            if (defType.Context.Target.Architecture == TargetArchitecture.ARM)
             {
+                layoutFromMetadata.LayoutAbiStable = false; // Int128 parameter passing ABI is unstable at this time
                 return layoutFromMetadata;
             }
 
@@ -42,7 +44,7 @@ public override ComputedInstanceFieldLayout ComputeInstanceLayout(DefType defTyp
                 FieldAlignment = new LayoutInt(16),
                 FieldSize = layoutFromMetadata.FieldSize,
                 Offsets = layoutFromMetadata.Offsets,
-                LayoutAbiStable = true
+                LayoutAbiStable = false // Int128 parameter passing ABI is unstable at this time
             };
         }
 
@@ -72,7 +74,7 @@ public override ValueTypeShapeCharacteristics ComputeValueTypeShapeCharacteristi
         public static bool IsIntegerType(DefType type)
         {
             return type.IsIntrinsic
-                && type.Namespace == "System."
+                && type.Namespace == "System"
                 && ((type.Name == "Int128") || (type.Name == "UInt128"));
         }
     }

src/coreclr/tools/Common/TypeSystem/Interop/IL/MarshalHelpers.cs

@@ -434,6 +434,12 @@ internal static MarshallerKind GetMarshallerKind(
                     return MarshallerKind.Invalid;
                 }
 
+                if (!isField && InteropTypes.IsInt128Type(context, type))
+                {
+                    // Int128 types cannot be passed by value
+                    return MarshallerKind.Invalid;
+                }
+
                 if (isBlittable)
                 {
                     if (nativeType != NativeTypeKind.Default && nativeType != NativeTypeKind.Struct)

src/coreclr/tools/Common/TypeSystem/Interop/InteropTypes.cs

@@ -137,6 +137,11 @@ public static bool IsSystemRuntimeIntrinsicsVector64T(TypeSystemContext context,
             return IsCoreNamedType(context, type, "System.Runtime.Intrinsics", "Vector64`1");
         }
 
+        public static bool IsInt128Type(TypeSystemContext context, TypeDesc type)
+        {
+            return IsCoreNamedType(context, type, "System", "Int128") || IsCoreNamedType(context, type, "System", "UInt128");
+        }
+
         public static bool IsSystemRuntimeIntrinsicsVector128T(TypeSystemContext context, TypeDesc type)
         {
             return IsCoreNamedType(context, type, "System.Runtime.Intrinsics", "Vector128`1");

src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/ArchitectureSpecificFieldLayoutTests.cs

@@ -19,9 +19,16 @@ public class ArchitectureSpecificFieldLayoutTests
         ModuleDesc _testModuleX86;
         TestTypeSystemContext _contextX64;
         ModuleDesc _testModuleX64;
+        TestTypeSystemContext _contextX64Windows;
+        ModuleDesc _testModuleX64Windows;
+        TestTypeSystemContext _contextX64Linux;
+        ModuleDesc _testModuleX64Linux;
         TestTypeSystemContext _contextARM;
         ModuleDesc _testModuleARM;
 
+        TestTypeSystemContext _contextARM64;
+        ModuleDesc _testModuleARM64;
+
         public ArchitectureSpecificFieldLayoutTests()
         {
             _contextX64 = new TestTypeSystemContext(TargetArchitecture.X64);
@@ -30,6 +37,18 @@ public ArchitectureSpecificFieldLayoutTests()
 
             _testModuleX64 = systemModuleX64;
 
+            _contextX64Linux = new TestTypeSystemContext(TargetArchitecture.X64, TargetOS.Linux);
+            var systemModuleX64Linux = _contextX64Linux.CreateModuleForSimpleName("CoreTestAssembly");
+            _contextX64Linux.SetSystemModule(systemModuleX64Linux);
+
+            _testModuleX64Linux = systemModuleX64Linux;
+
+            _contextX64Windows = new TestTypeSystemContext(TargetArchitecture.X64, TargetOS.Windows);
+            var systemModuleX64Windows = _contextX64Windows.CreateModuleForSimpleName("CoreTestAssembly");
+            _contextX64Windows.SetSystemModule(systemModuleX64Windows);
+
+            _testModuleX64Windows = systemModuleX64Windows;
+
             _contextARM = new TestTypeSystemContext(TargetArchitecture.ARM);
             var systemModuleARM = _contextARM.CreateModuleForSimpleName("CoreTestAssembly");
             _contextARM.SetSystemModule(systemModuleARM);
@@ -41,6 +60,12 @@ public ArchitectureSpecificFieldLayoutTests()
             _contextX86.SetSystemModule(systemModuleX86);
 
             _testModuleX86 = systemModuleX86;
+
+            _contextARM64 = new TestTypeSystemContext(TargetArchitecture.ARM64);
+            var systemModuleARM64 = _contextARM64.CreateModuleForSimpleName("CoreTestAssembly");
+            _contextARM64.SetSystemModule(systemModuleARM64);
+
+            _testModuleARM64 = systemModuleARM64;
         }
 
         [Fact]
@@ -476,5 +501,54 @@ public void TestAlignmentBehavior_AutoAlignmentRules(string wrapperType, int[] a
             Assert.Equal(alignment[2], tX86.GetField("fld2").Offset.AsInt);
         }
 
+        [Theory]
+        [InlineData("StructStructByte_Int128StructAuto", "ARM64", 16, 32)]
+        [InlineData("StructStructByte_Int128StructAuto", "ARM", 8, 24)]
+        [InlineData("StructStructByte_Int128StructAuto", "X86", 16, 32)]
+        [InlineData("StructStructByte_Int128StructAuto", "X64Linux", 16, 32)]
+        [InlineData("StructStructByte_Int128StructAuto", "X64Windows", 16, 32)]
+        [InlineData("StructStructByte_UInt128StructAuto", "ARM64", 16, 32)]
+        [InlineData("StructStructByte_UInt128StructAuto", "ARM", 8, 24)]
+        [InlineData("StructStructByte_UInt128StructAuto", "X86", 16, 32)]
+        [InlineData("StructStructByte_UInt128StructAuto", "X64Linux", 16, 32)]
+        [InlineData("StructStructByte_UInt128StructAuto", "X64Windows", 16, 32)]
+        // Variation of TestAlignmentBehavior_AutoAlignmentRules above that is able to deal with os specific behavior
+        public void TestAlignmentBehavior_AutoAlignmentRulesWithOSDependence(string wrapperType, string osArch, int alignment, int size)
+        {
+            ModuleDesc testModule;
+            switch (osArch)
+            {
+                case "ARM64":
+                    testModule = _testModuleARM64;
+                    break;
+                case "ARM":
+                    testModule = _testModuleARM;
+                    break;
+                case "X64":
+                    testModule = _testModuleX64;
+                    break;
+                case "X64Linux":
+                    testModule = _testModuleX64Linux;
+                    break;
+                case "X64Windows":
+                    testModule = _testModuleX64Windows;
+                    break;
+                case "X86":
+                    testModule = _testModuleX86;
+                    break;
+                default:
+                    throw new Exception();
+            }
+
+            string _namespace = "Sequential";
+            string _type = wrapperType;
+
+            MetadataType type = testModule.GetType(_namespace, _type);
+
+            Assert.Equal(alignment, type.InstanceFieldAlignment.AsInt);
+            Assert.Equal(size, type.InstanceFieldSize.AsInt);
+            Assert.Equal(0x0, type.GetField("fld1").Offset.AsInt);
+            Assert.Equal(alignment, type.GetField("fld2").Offset.AsInt);
+        }
     }
 }

src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/CoreTestAssembly/InstanceFieldLayout.cs

@@ -212,6 +212,18 @@ public struct StructStructByte_Struct9BytesAuto
         public StructByte fld1;
         public Auto.Struct9Bytes fld2;
     }
+
+    public struct StructStructByte_Int128StructAuto
+    {
+        public StructByte fld1;
+        public Auto.Int128Struct fld2;
+    }
+
+    public struct StructStructByte_UInt128StructAuto
+    {
+        public StructByte fld1;
+        public Auto.UInt128Struct fld2;
+    }
 }
 
 namespace Auto
@@ -417,6 +429,18 @@ public struct Struct9Bytes
         public byte fld8;
         public byte fld9;
     }
+
+    [StructLayout(LayoutKind.Auto)]
+    public struct UInt128Struct
+    {
+        UInt128 fld1;
+    }
+
+    [StructLayout(LayoutKind.Auto)]
+    public struct Int128Struct
+    {
+        Int128 fld1;
+    }
 }
 
 namespace IsByRefLike

src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/CoreTestAssembly/Platform.cs

@@ -73,6 +73,24 @@ public ref struct TypedReference
         private readonly ref byte _value;
         private readonly RuntimeTypeHandle _typeHandle;
     }
+
+    [Intrinsic]
+    [StructLayout(LayoutKind.Sequential)]
+    public readonly struct Int128
+    {
+
+        private readonly ulong _lower;
+        private readonly ulong _upper;
+    }
+
+    [Intrinsic]
+    [StructLayout(LayoutKind.Sequential)]
+    public readonly struct UInt128
+    {
+
+        private readonly ulong _lower;
+        private readonly ulong _upper;
+    }
 }
 
 namespace System.Collections

src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/ILCompiler.TypeSystem.Tests.csproj

@@ -47,6 +47,7 @@
     <Compile Include="../../Common/TypeSystem/Interop/InteropTypes.cs" Link="TypeSystem/Interop/InteropTypes.cs" />
     <Compile Include="../../Common/TypeSystem/IL/HelperExtensions.cs" Link="TypeSystem/IL/HelperExtensions" />
     <Compile Include="../../Common/Compiler/VectorFieldLayoutAlgorithm.cs" Link="Compiler/VectorFieldLayoutAlgorithm.cs" />
+    <Compile Include="../../Common/Compiler/Int128FieldLayoutAlgorithm.cs" Link="Compiler/Int128FieldLayoutAlgorithm.cs" />
     <Compile Include="ArchitectureSpecificFieldLayoutTests.cs" />
     <Compile Include="CanonicalizationTests.cs" />
     <Compile Include="ConstraintsValidationTest.cs" />

src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/TestTypeSystemContext.cs

@@ -24,17 +24,20 @@ class TestTypeSystemContext : MetadataTypeSystemContext
         Dictionary<string, ModuleDesc> _modules = new Dictionary<string, ModuleDesc>(StringComparer.OrdinalIgnoreCase);
 
         private VectorFieldLayoutAlgorithm _vectorFieldLayoutAlgorithm;
+        private Int128FieldLayoutAlgorithm _int128FieldLayoutAlgorithm;
+
         MetadataFieldLayoutAlgorithm _metadataFieldLayout = new TestMetadataFieldLayoutAlgorithm();
         MetadataRuntimeInterfacesAlgorithm _metadataRuntimeInterfacesAlgorithm = new MetadataRuntimeInterfacesAlgorithm();
         ArrayOfTRuntimeInterfacesAlgorithm _arrayOfTRuntimeInterfacesAlgorithm;
         VirtualMethodAlgorithm _virtualMethodAlgorithm = new MetadataVirtualMethodAlgorithm();
 
         public CanonicalizationMode CanonMode { get; set; } = CanonicalizationMode.RuntimeDetermined;
 
-        public TestTypeSystemContext(TargetArchitecture arch)
-            : base(new TargetDetails(arch, TargetOS.Unknown, TargetAbi.Unknown))
+        public TestTypeSystemContext(TargetArchitecture arch, TargetOS targetOS = TargetOS.Unknown)
+            : base(new TargetDetails(arch, targetOS, TargetAbi.Unknown))
         {
             _vectorFieldLayoutAlgorithm = new VectorFieldLayoutAlgorithm(_metadataFieldLayout, true);
+            _int128FieldLayoutAlgorithm = new Int128FieldLayoutAlgorithm(_metadataFieldLayout);
         }
 
         public ModuleDesc GetModuleForSimpleName(string simpleName)
@@ -74,6 +77,10 @@ public override FieldLayoutAlgorithm GetLayoutAlgorithmForType(DefType type)
             {
                 return _vectorFieldLayoutAlgorithm;
             }
+            else if (Int128FieldLayoutAlgorithm.IsIntegerType(type))
+            {
+                return _int128FieldLayoutAlgorithm;
+            }
 
             return _metadataFieldLayout;
         }

src/coreclr/vm/methodtablebuilder.cpp

@@ -9907,21 +9907,6 @@ void MethodTableBuilder::CheckForSystemTypes()
 
                 return;
             }
-#if defined(UNIX_AMD64_ABI) || defined(TARGET_ARM64)
-            else if (strcmp(nameSpace, g_SystemNS) == 0)
-            {
-                EEClassLayoutInfo* pLayout = pClass->GetLayoutInfo();
-
-                // These types correspond to fundamental data types in the underlying ABIs:
-                // * Int128:  __int128
-                // * UInt128: unsigned __int128
-
-                if ((strcmp(name, g_Int128Name) == 0) || (strcmp(name, g_UInt128Name) == 0))
-                {
-                    pLayout->m_ManagedLargestAlignmentRequirementOfAllMembers = 16; // sizeof(__int128)
-                }
-            }
-#endif // UNIX_AMD64_ABI || TARGET_ARM64
         }
 
         if (g_pNullableClass != NULL)
@@ -10005,6 +9990,29 @@ void MethodTableBuilder::CheckForSystemTypes()
         {
             pMT->SetInternalCorElementType (ELEMENT_TYPE_I);
         }
+        else if ((strcmp(name, g_Int128Name) == 0) || (strcmp(name, g_UInt128Name) == 0))
+        {
+            EEClassLayoutInfo* pLayout = pClass->GetLayoutInfo();
+#ifdef TARGET_ARM
+            // No such type exists for the Procedure Call Standard for ARM. We will default
+            // to the same alignment as __m128, which is supported by the ABI.
+
+            pLayout->m_ManagedLargestAlignmentRequirementOfAllMembers = 8;
+#elif defined(TARGET_64BIT) || defined(TARGET_X86)
+
+            // These types correspond to fundamental data types in the underlying ABIs:
+            // * Int128:  __int128
+            // * UInt128: unsigned __int128
+            //
+            // This behavior matches the ABI standard on various Unix platforms
+            // On Windows, no standard for Int128 has been established yet, 
+            // although applying 16 byte alignment is consistent with treatment of 128 bit SSE types
+            // even on X86
+            pLayout->m_ManagedLargestAlignmentRequirementOfAllMembers = 16; // sizeof(__int128)
+#else
+#error Unknown architecture
+#endif // TARGET_64BIT
+        }
     }
     else
     {

src/coreclr/vm/mlinfo.cpp

@@ -2283,6 +2283,21 @@ MarshalInfo::MarshalInfo(Module* pModule,
                     IfFailGoto(E_FAIL, lFail);
                 }
 
+                // * Int128: Represents the 128 bit integer ABI primitive type which requires currently unimplemented handling
+                // * UInt128: Represents the 128 bit integer ABI primitive type which requires currently unimplemented handling
+                // The field layout is correct, so field scenarios work, but these should not be passed by value as parameters
+                //
+                // TODO: Contact Jeremy to figure out how to prohibit these types from being used as fields of parameters passed by value
+                if (!IsFieldScenario())
+                {
+                    if (m_pMT == CoreLibBinder::GetClass(CLASS__INT128)
+                        || m_pMT == CoreLibBinder::GetClass(CLASS__UINT128))
+                    {
+                        m_resID = IDS_EE_BADMARSHAL_INT128_RESTRICTION;
+                        IfFailGoto(E_FAIL, lFail);
+                    }
+                }
+
                 if (!m_pMT->HasLayout())
                 {
                     m_resID = IDS_EE_BADMARSHAL_AUTOLAYOUT;