Start building a universal System.Linq.Expressions library (dotnet#61952

)

System.Linq.Expressions currently offers multiple build time definitions to build different flavors of the library:

* `FEATURE_COMPILE` (defined everywhere except iOS/tvOS/Catalyst, and NativeAOT) - enables Reflection.Emit-based expression tree compiler.
* `FEATURE_INTERPRET` (always defined and not actually possible to build without) - enables the expression interpreter
* `FEATURE_DLG_INVOKE` (defined everywhere except NativeAOT, but we likely need to be able to run without it on iOS too because there's uninvestigated bugs around it ActiveIssue'd in dotnet#54970) - in the interpreter, use a delegate to invoke `CallInstructions` instead of `MethodInfo.Invoke`. The delegate might have to be Reflection.Emitted if it's not supportable with `Func`/`Action` (that's the uninvestigated iOS/tvOS/Catalyst bug).

For dotnet#61231, we need to be able to build a single System.Linq.Expression library that can switch between these build-time configurations _at publish time_ since we don't want to build a separate S.L.Expressions library for NativeAOT. There are advantages in having this setup for non-NativeAOT scenarios too. This pull request accomplishes that by mechanically changing the `#define`s into feature switches.

The feature switch is placed in the last proposed location for dotnet#17973. I expect we'll want such API to be public at some point; now that Xamarin and NativeAOT use this formerly .NET Native-only thing the API request became relevant again.

This pull request is focused on the mechanical replacement of `#defines` with feature switches and it's already a lot bigger than I'm comfortable with.

There's some obvious "`!FEATURE_COMPILE` means this is .NET Native with everything what that meant" that I did not touch because this is meant to be a mechanical change. Some cleanup will be needed at some point. Right now this just mostly means we're running fewer tests than we could.

Validation:
* Verified that we're still running the same number of tests with CoreCLR as we previously were and they're all passing.
* Verified we're getting mostly the same size of the S.L.Expressions library on iOS (433 kB grew to 436 kB, the diffs are expected).
* Verified things work on the NativeAOT side as well.

eng/illink.targets

@@ -236,6 +236,7 @@
       <ILLinkArgs Condition="'$(ILLinkRewritePDBs)' == 'true' and Exists('$(ILLinkTrimAssemblySymbols)')">$(ILLinkArgs) -b true</ILLinkArgs>
       <!-- pass the non-embedded descriptors xml file on the command line -->
       <ILLinkArgs Condition="'$(ILLinkDescriptorsLibraryBuildXml)' != ''">$(ILLinkArgs) -x "$(ILLinkDescriptorsLibraryBuildXml)"</ILLinkArgs>
+      <ILLinkArgs Condition="'$(ILLinkSubstitutionsLibraryBuildXml)' != ''">$(ILLinkArgs) --substitutions "$(ILLinkSubstitutionsLibraryBuildXml)"</ILLinkArgs>
       <!-- suppress warnings with the following codes:
            IL2008: Could not find type A specified in resource B
            IL2009: Could not find method A in type B specified in resource C
@@ -254,6 +255,7 @@
       <!-- IL2067-IL2091: Unsatisfied DynamicallyAccessedMembers requirements -->
       <LinkerNoWarn>$(LinkerNoWarn);IL2067;IL2068;IL2069;IL2070;IL2071;IL2072;IL2073;IL2074;IL2075;IL2076;IL2077;IL2078;IL2079;IL2080;IL2081;IL2082;IL2083;IL2084;IL2085;IL2086;IL2087;IL2088;IL2089;IL2090;IL2091</LinkerNoWarn>
       <ILLinkArgs>$(ILLinkArgs) --nowarn $(LinkerNoWarn)</ILLinkArgs>
+      <ILLinkArgs Condition="'$(ILLinkDisableIPConstProp)' == 'true'">$(ILLinkArgs) --disable-opt ipconstprop</ILLinkArgs>
     </PropertyGroup>
 
     <MakeDir Directories="$(ILLinkTrimInputPath)" />

src/libraries/Common/tests/TestUtilities/System/PlatformDetection.cs

@@ -82,15 +82,7 @@ public static partial class PlatformDetection
         private static readonly Lazy<bool> s_LinqExpressionsBuiltWithIsInterpretingOnly = new Lazy<bool>(GetLinqExpressionsBuiltWithIsInterpretingOnly);
         private static bool GetLinqExpressionsBuiltWithIsInterpretingOnly()
         {
-            Type type = typeof(LambdaExpression);
-            if (type != null)
-            {
-                // The "Accept" method is under FEATURE_COMPILE conditional so it should not exist
-                MethodInfo methodInfo = type.GetMethod("Accept", BindingFlags.NonPublic | BindingFlags.Static);
-                return methodInfo == null;
-            }
-
-            return false;
+            return !(bool)typeof(LambdaExpression).GetMethod("get_CanCompileToIL").Invoke(null, Array.Empty<object>());
         }
 
         // Please make sure that you have the libgdiplus dependency installed.

src/libraries/System.Linq.Expressions/src/ILLink/ILLink.Substitutions.IsInterpreting.LibraryBuild.xml

@@ -0,0 +1,7 @@
+<linker>
+  <assembly fullname="System.Linq.Expressions">
+    <type fullname="System.Linq.Expressions.LambdaExpression">
+      <method signature="System.Boolean get_CanCompileToIL()" body="stub" value="false" />
+    </type>
+  </assembly>
+</linker>

src/libraries/System.Linq.Expressions/src/ILLink/ILLink.Substitutions.xml

@@ -0,0 +1,13 @@
+<linker>
+  <assembly fullname="System.Linq.Expressions">
+    <type fullname="System.Linq.Expressions.LambdaExpression">
+      <method signature="System.Boolean get_CanCompileToIL()" feature="System.Linq.Expressions.CanCompileToIL" featurevalue="false" body="stub" value="false" />
+    </type>
+    <type fullname="System.Dynamic.Utils.DelegateHelpers">
+      <method signature="System.Boolean get_CanEmitObjectArrayDelegate()" feature="System.Linq.Expressions.CanEmitObjectArrayDelegate" featurevalue="false" body="stub" value="false" />
+    </type>
+    <type fullname="System.Linq.Expressions.Interpreter.CallInstruction">
+      <method signature="System.Boolean get_CanCreateArbitraryDelegates()" feature="System.Linq.Expressions.CanCreateArbitraryDelegates" featurevalue="false" body="stub" value="false" />
+    </type>
+  </assembly>
+</linker>

src/libraries/System.Linq.Expressions/src/MatchingRefApiCompatBaseline.txt

@@ -15,4 +15,6 @@ TypesMustExist : Type 'System.Linq.Expressions.Interpreter.LightLambda' does not
 TypesMustExist : Type 'System.Runtime.CompilerServices.CallSiteOps' does not exist in the reference but it does exist in the implementation.
 TypesMustExist : Type 'System.Runtime.CompilerServices.Closure' does not exist in the reference but it does exist in the implementation.
 TypesMustExist : Type 'System.Runtime.CompilerServices.RuntimeOps' does not exist in the reference but it does exist in the implementation.
-Total Issues: 16
+MembersMustExist : Member 'public System.Boolean System.Linq.Expressions.LambdaExpression.CanCompileToIL.get()' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'public System.Boolean System.Linq.Expressions.LambdaExpression.CanInterpret.get()' does not exist in the reference but it does exist in the implementation.
+Total Issues: 18

src/libraries/System.Linq.Expressions/src/System.Linq.Expressions.csproj

@@ -1,14 +1,22 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <FeatureInterpret>true</FeatureInterpret>
     <TargetFrameworks>$(NetCoreAppCurrent);$(NetCoreAppCurrent)-iOS;$(NetCoreAppCurrent)-tvOS;$(NetCoreAppCurrent)-MacCatalyst</TargetFrameworks>
     <Nullable>enable</Nullable>
     <IsInterpreting>false</IsInterpreting>
     <IsInterpreting Condition="'$(TargetsiOS)' == 'true' or '$(TargetstvOS)' == 'true' or '$(TargetsMacCatalyst)' == 'true'">true</IsInterpreting>
-    <DefineConstants> $(DefineConstants);FEATURE_DLG_INVOKE;FEATURE_FAST_CREATE</DefineConstants>
-    <DefineConstants Condition=" '$(IsInterpreting)' != 'true' ">$(DefineConstants);FEATURE_COMPILE</DefineConstants>
-    <DefineConstants Condition=" '$(FeatureInterpret)' == 'true' ">$(DefineConstants);FEATURE_INTERPRET</DefineConstants>
+    <ILLinkSubstitutionsLibraryBuildXml Condition="'$(IsInterpreting)' == 'true'">ILLink\ILLink.Substitutions.IsInterpreting.LibraryBuild.xml</ILLinkSubstitutionsLibraryBuildXml>
+    <DefineConstants> $(DefineConstants);FEATURE_FAST_CREATE</DefineConstants>
+
+    <!--
+      Disable constant propagation so that methods referenced from ILLink.Substitutions.xml don't get inlined
+      with a wrong value at library build time and the substitution can still be selected at publish time.
+      For iOS/tvOS/Catalyst we prefer smaller size by default, so keep constprop enabled to get rid of the expression compiler.
+    -->
+    <ILLinkDisableIPConstProp Condition="'$(TargetsiOS)' != 'true' and '$(TargetstvOS)' != 'true' and '$(TargetsMacCatalyst)' != 'true'">true</ILLinkDisableIPConstProp>
   </PropertyGroup>
+  <ItemGroup>
+    <ILLinkSubstitutionsXmls Include="$(ILLinkDirectory)ILLink.Substitutions.xml" />
+  </ItemGroup>
   <ItemGroup>
     <Compile Include="$(CommonPath)System\Obsoletions.cs"
              Link="Common\System\Obsoletions.cs" />
@@ -125,7 +133,7 @@
     <Compile Include="System\Dynamic\UnaryOperationBinder.cs" />
     <Compile Include="System\Dynamic\IInvokeOnGetBinder.cs" />
   </ItemGroup>
-  <ItemGroup Condition=" '$(IsInterpreting)' != 'true'">
+  <ItemGroup>
     <Compile Include="System\Linq\Expressions\Compiler\AssemblyGen.cs" />
     <Compile Include="System\Dynamic\Utils\Helpers.cs" />
     <Compile Include="System\Linq\Expressions\Compiler\AnalyzedTree.cs" />
@@ -157,7 +165,7 @@
     <Compile Include="System\Runtime\CompilerServices\RuntimeOps.ExpressionQuoter.cs" />
     <Compile Include="System\Runtime\CompilerServices\RuntimeOps.RuntimeVariableList.cs" />
   </ItemGroup>
-  <ItemGroup Condition="'$(IsInterpreting)' == 'true' Or '$(FeatureInterpret)' == 'true'">
+  <ItemGroup>
     <Compile Include="System\Dynamic\Utils\DelegateHelpers.cs" />
     <Compile Include="System\Linq\Expressions\Interpreter\AddInstruction.cs" />
     <Compile Include="System\Linq\Expressions\Interpreter\AndInstruction.cs" />

src/libraries/System.Linq.Expressions/src/System/Dynamic/UpdateDelegates.Generated.cs

@@ -7,7 +7,6 @@ namespace System.Dynamic
 {
     internal static partial class UpdateDelegates
     {
-#if FEATURE_COMPILE
         [Obsolete("pregenerated CallSite<T>.Update delegate", error: true)]
         internal static TRet UpdateAndExecute1<T0, TRet>(CallSite site, T0 arg0)
         {
@@ -2895,6 +2894,5 @@ internal static void NoMatchVoid10<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>(CallS
             site._match = false;
             return;
         }
-#endif
     }
 }

src/libraries/System.Linq.Expressions/src/System/Dynamic/UpdateDelegates.Generated.tt

@@ -13,7 +13,6 @@ namespace System.Dynamic
 {
     internal static partial class UpdateDelegates
     {
-#if FEATURE_COMPILE
 <#
 for (int i = 1; i <= 10; i++)
 {
@@ -330,6 +329,5 @@ for (int i = 1; i <= 10; i++)
     }
 }
 #>
-#endif
     }
 }

src/libraries/System.Linq.Expressions/src/System/Dynamic/Utils/DelegateHelpers.cs

@@ -3,28 +3,43 @@
 
 using System.Diagnostics.CodeAnalysis;
 using System.Reflection;
-
-#if !FEATURE_DYNAMIC_DELEGATE
 using System.Reflection.Emit;
 using System.Text;
 using System.Threading;
-#endif
 
 namespace System.Dynamic.Utils
 {
     internal static class DelegateHelpers
     {
-        internal static Delegate CreateObjectArrayDelegate(Type delegateType, Func<object?[], object?> handler)
+        // This can be flipped to true using feature switches at publishing time
+        internal static bool CanEmitObjectArrayDelegate => true;
+
+        // Separate class so that the it can be trimmed away and doesn't get conflated
+        // with the Reflection.Emit statics below.
+        private static class DynamicDelegateLightup
         {
-#if !FEATURE_DYNAMIC_DELEGATE
-            return CreateObjectArrayDelegateRefEmit(delegateType, handler);
-#else
-            return Internal.Runtime.Augments.DynamicDelegateAugments.CreateObjectArrayDelegate(delegateType, handler);
-#endif
+            public static Func<Type, Func<object?[], object?>, Delegate> CreateObjectArrayDelegate { get; }
+                = CreateObjectArrayDelegateInternal();
+
+            [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2075:UnrecognizedReflectionPattern",
+                Justification = "Works around https://github.com/dotnet/linker/issues/2392")]
+            private static Func<Type, Func<object?[], object?>, Delegate> CreateObjectArrayDelegateInternal()
+                => Type.GetType("Internal.Runtime.Augments.DynamicDelegateAugments")!
+                    .GetMethod("CreateObjectArrayDelegate")!
+                    .CreateDelegate<Func<Type, Func<object?[], object?>, Delegate>>();
         }
 
-
-#if !FEATURE_DYNAMIC_DELEGATE
+        internal static Delegate CreateObjectArrayDelegate(Type delegateType, Func<object?[], object?> handler)
+        {
+            if (CanEmitObjectArrayDelegate)
+            {
+                return CreateObjectArrayDelegateRefEmit(delegateType, handler);
+            }
+            else
+            {
+                return DynamicDelegateLightup.CreateObjectArrayDelegate(delegateType, handler);
+            }
+        }
 
         private static readonly CacheDict<Type, MethodInfo> s_thunks = new CacheDict<Type, MethodInfo>(256);
         private static readonly MethodInfo s_FuncInvoke = typeof(Func<object?[], object?>).GetMethod("Invoke")!;
@@ -290,7 +305,5 @@ private static Type ConvertToBoxableType(Type t)
         {
             return (t.IsPointer) ? typeof(IntPtr) : t;
         }
-
-#endif
     }
 }

src/libraries/System.Linq.Expressions/src/System/Dynamic/Utils/TypeExtensions.cs

@@ -80,7 +80,6 @@ internal static ParameterInfo[] GetParametersCached(this MethodBase method)
             return pis;
         }
 
-#if FEATURE_COMPILE
         // Expression trees/compiler just use IsByRef, why do we need this?
         // (see LambdaCompiler.EmitArguments for usage in the compiler)
         internal static bool IsByRefParameter(this ParameterInfo pi)
@@ -91,6 +90,5 @@ internal static bool IsByRefParameter(this ParameterInfo pi)
 
             return (pi.Attributes & ParameterAttributes.Out) == ParameterAttributes.Out;
         }
-#endif
     }
 }

src/libraries/System.Linq.Expressions/src/System/Dynamic/Utils/TypeUtils.cs

@@ -911,8 +911,6 @@ public static MethodInfo GetInvokeMethod(this Type delegateType)
             return delegateType.GetMethod("Invoke", BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)!;
         }
 
-#if FEATURE_COMPILE
-
         internal static bool IsUnsigned(this Type type) => IsUnsigned(GetNonNullableType(type).GetTypeCode());
 
         internal static bool IsUnsigned(this TypeCode typeCode)
@@ -946,8 +944,6 @@ internal static bool IsFloatingPoint(this TypeCode typeCode)
             }
         }
 
-#endif
-
         [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2070:UnrecognizedReflectionPattern",
             Justification = "The Array 'Get' method is dynamically constructed and is not included in IL. It is not subject to trimming.")]
         public static MethodInfo GetArrayGetMethod(Type arrayType)

src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/Common/CachedReflectionInfo.cs

@@ -159,7 +159,6 @@ public static FieldInfo DateTime_MinValue
                                  (s_Math_Pow_Double_Double = typeof(Math).GetMethod(nameof(Math.Pow), new[] { typeof(double), typeof(double) })!);
 
         // Closure and RuntimeOps helpers are used only in the compiler.
-#if FEATURE_COMPILE
         private static ConstructorInfo? s_Closure_ObjectArray_ObjectArray;
         public static ConstructorInfo Closure_ObjectArray_ObjectArray =>
                                        s_Closure_ObjectArray_ObjectArray ??
@@ -194,6 +193,5 @@ public static FieldInfo DateTime_MinValue
         public static MethodInfo RuntimeOps_Quote =>
                                   s_RuntimeOps_Quote ??
                                  (s_RuntimeOps_Quote = typeof(RuntimeOps).GetMethod(nameof(RuntimeOps.Quote))!);
-#endif
     }
 }

src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/Compiler/DelegateHelpers.Generated.cs

@@ -84,8 +84,6 @@ private static TypeInfo NextTypeInfo(Type initialArg, TypeInfo curTypeInfo)
             return nextTypeInfo;
         }
 
-#if !FEATURE_COMPILE
-
         public delegate object VBCallSiteDelegate0<T>(T callSite, object instance);
         public delegate object VBCallSiteDelegate1<T>(T callSite, object instance, ref object arg1);
         public delegate object VBCallSiteDelegate2<T>(T callSite, object instance, ref object arg1, ref object arg2);
@@ -95,7 +93,6 @@ private static TypeInfo NextTypeInfo(Type initialArg, TypeInfo curTypeInfo)
         public delegate object VBCallSiteDelegate6<T>(T callSite, object instance, ref object arg1, ref object arg2, ref object arg3, ref object arg4, ref object arg5, ref object arg6);
         public delegate object VBCallSiteDelegate7<T>(T callSite, object instance, ref object arg1, ref object arg2, ref object arg3, ref object arg4, ref object arg5, ref object arg6, ref object arg7);
 
-
         private static Type TryMakeVBStyledCallSite(Type[] types)
         {
             // Shape of VB CallSiteDelegates is CallSite * (instance : obj) * [arg-n : byref obj] -> obj
@@ -128,7 +125,6 @@ private static Type TryMakeVBStyledCallSite(Type[] types)
                 default: return null;
             }
         }
-#endif
 
         /// <summary>
         /// Creates a new delegate, or uses a func/action
@@ -163,11 +159,14 @@ internal static Type MakeNewDelegate(Type[] types)
 
             if (needCustom)
             {
-#if FEATURE_COMPILE
-                return MakeNewCustomDelegate(types);
-#else
-                return TryMakeVBStyledCallSite(types) ?? MakeNewCustomDelegate(types);
-#endif
+                if (LambdaExpression.CanCompileToIL)
+                {
+                    return MakeNewCustomDelegate(types);
+                }
+                else
+                {
+                    return TryMakeVBStyledCallSite(types) ?? MakeNewCustomDelegate(types);
+                }
             }
 
             Type result;

src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/Compiler/DelegateHelpers.cs

@@ -107,26 +107,27 @@ private static bool IsByRef(DynamicMetaObject mo)
             return mo.Expression is ParameterExpression pe && pe.IsByRef;
         }
 
-#if FEATURE_COMPILE
-        private const MethodAttributes CtorAttributes = MethodAttributes.RTSpecialName | MethodAttributes.HideBySig | MethodAttributes.Public;
-        private const MethodImplAttributes ImplAttributes = MethodImplAttributes.Runtime | MethodImplAttributes.Managed;
-        private const MethodAttributes InvokeAttributes = MethodAttributes.Public | MethodAttributes.HideBySig | MethodAttributes.NewSlot | MethodAttributes.Virtual;
-        private static readonly Type[] s_delegateCtorSignature = { typeof(object), typeof(IntPtr) };
-#endif
-
         private static Type MakeNewCustomDelegate(Type[] types)
         {
-#if FEATURE_COMPILE
-            Type returnType = types[types.Length - 1];
-            Type[] parameters = types.RemoveLast();
-
-            TypeBuilder builder = AssemblyGen.DefineDelegateType("Delegate" + types.Length);
-            builder.DefineConstructor(CtorAttributes, CallingConventions.Standard, s_delegateCtorSignature).SetImplementationFlags(ImplAttributes);
-            builder.DefineMethod("Invoke", InvokeAttributes, returnType, parameters).SetImplementationFlags(ImplAttributes);
-            return builder.CreateTypeInfo()!;
-#else
-            throw new PlatformNotSupportedException();
-#endif
+            if (LambdaExpression.CanCompileToIL)
+            {
+                Type returnType = types[types.Length - 1];
+                Type[] parameters = types.RemoveLast();
+                Type[] delegateCtorSignature = { typeof(object), typeof(IntPtr) };
+
+                const MethodAttributes ctorAttributes = MethodAttributes.RTSpecialName | MethodAttributes.HideBySig | MethodAttributes.Public;
+                const MethodImplAttributes implAttributes = MethodImplAttributes.Runtime | MethodImplAttributes.Managed;
+                const MethodAttributes invokeAttributes = MethodAttributes.Public | MethodAttributes.HideBySig | MethodAttributes.NewSlot | MethodAttributes.Virtual;
+
+                TypeBuilder builder = AssemblyGen.DefineDelegateType("Delegate" + types.Length);
+                builder.DefineConstructor(ctorAttributes, CallingConventions.Standard, delegateCtorSignature).SetImplementationFlags(implAttributes);
+                builder.DefineMethod("Invoke", invokeAttributes, returnType, parameters).SetImplementationFlags(implAttributes);
+                return builder.CreateTypeInfo()!;
+            }
+            else
+            {
+                throw new PlatformNotSupportedException();
+            }
         }
     }
 }

src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/Interpreter/CallInstruction.Generated.cs

@@ -13,9 +13,7 @@ namespace System.Linq.Expressions.Interpreter
 {
     internal partial class CallInstruction
     {
-#if FEATURE_DLG_INVOKE
         private const int MaxHelpers = 5;
-#endif
 
 #if FEATURE_FAST_CREATE
         private const int MaxArgs = 3;
@@ -158,7 +156,6 @@ private static CallInstruction FastCreate<T0, T1>(MethodInfo target, ParameterIn
         }
 #endif
 
-#if FEATURE_DLG_INVOKE
         [return: DynamicallyAccessedMembersAttribute(DynamicallyAccessedMemberTypes.PublicConstructors)]
         private static Type GetHelperType(MethodInfo info, Type[] arrTypes)
         {
@@ -211,10 +208,8 @@ private static Type GetHelperType(MethodInfo info, Type[] arrTypes)
             }
             return t;
         }
-#endif
     }
 
-#if FEATURE_DLG_INVOKE
     internal sealed class ActionCallInstruction : CallInstruction
     {
         private readonly Action _target;
@@ -577,6 +572,4 @@ public override int Run(InterpretedFrame frame)
 
         public override string ToString() => "Call(" + _target.Method + ")";
     }
-
-#endif
 }