Prototype private runtime references with System.Reflection.Metadata (#560)

BuildScripts/IgnoresAccessChecksToGenerator/.gitignore

@@ -0,0 +1,4 @@
+artifacts/
+**/bin/
+**/obj/
+*.log

BuildScripts/IgnoresAccessChecksToGenerator/Consumer/Consumer.csproj

@@ -0,0 +1,26 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFrameworks>net8.0;net10.0</TargetFrameworks>
+    <OutputType>Exe</OutputType>
+    <RollForward>Disable</RollForward>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <FacadeDirectory Condition="'$(FacadeDirectory)' == ''">../artifacts/$(TargetFramework)/private</FacadeDirectory>
+    <GrantPrivateAccess Condition="'$(GrantPrivateAccess)' == ''">true</GrantPrivateAccess>
+    <DefineConstants Condition="'$(GrantPrivateAccess)' == 'true'">$(DefineConstants);GRANT_PRIVATE_ACCESS</DefineConstants>
+    <DefineConstants Condition="'$(MissingMemberProbe)' == 'true'">$(DefineConstants);MISSING_MEMBER</DefineConstants>
+    <DefineConstants Condition="'$(WrongSignatureProbe)' == 'true'">$(DefineConstants);WRONG_SIGNATURE</DefineConstants>
+    <DefineConstants Condition="'$(UseGeneratorTask)' == 'true'">$(DefineConstants);GENERATED_ACCESS_ATTRIBUTE</DefineConstants>
+  </PropertyGroup>
+  <ItemGroup>
+    <AssemblyAttribute Include="System.Runtime.Versioning.SupportedOSPlatformAttribute" Condition="'$(OS)' == 'Windows_NT'">
+      <_Parameter1>windows</_Parameter1>
+    </AssemblyAttribute>
+    <Reference Include="System.Private.CoreLib">
+      <HintPath>$(FacadeDirectory)/System.Private.CoreLib.dll</HintPath>
+      <Private>false</Private>
+    </Reference>
+  </ItemGroup>
+  <Import Project="../build/RuntimeReferences.targets" Condition="'$(UseGeneratorTask)' != 'true'" />
+  <Import Project="../build/IgnoresAccessChecksToGenerator.targets" Condition="'$(UseGeneratorTask)' == 'true'" />
+</Project>

BuildScripts/IgnoresAccessChecksToGenerator/Consumer/Program.cs

@@ -0,0 +1,61 @@
+using System.Reflection;
+using System.Reflection.Emit;
+using System.Runtime.CompilerServices;
+
+#if GRANT_PRIVATE_ACCESS && !GENERATED_ACCESS_ATTRIBUTE
+[assembly: IgnoresAccessChecksTo("System.Private.CoreLib")]
+#endif
+
+var actual = typeof(object).Assembly.Location;
+var expected = Environment.GetEnvironmentVariable("EXPECTED_CORELIB");
+if (expected != null && !string.Equals(Path.GetFullPath(actual), Path.GetFullPath(expected), StringComparison.OrdinalIgnoreCase))
+    throw new Exception("Consumer executed a different runtime than the selected reference: " + actual);
+if (!actual.StartsWith(System.Runtime.InteropServices.RuntimeEnvironment.GetRuntimeDirectory(), StringComparison.OrdinalIgnoreCase))
+    throw new Exception("Unexpected runtime CoreLib: " + actual);
+Console.WriteLine("Executing installed CoreLib: " + actual);
+var method = new DynamicMethod("direct_private_access", typeof(int), Type.EmptyTypes, typeof(Program), true);
+var il = (RuntimeILGenerator)method.GetILGenerator(16);
+Console.WriteLine($"Original visibility: stream={typeof(RuntimeILGenerator).GetField("m_ILStream", BindingFlags.Instance | BindingFlags.NonPublic)!.Attributes}; increase={typeof(RuntimeILGenerator).GetMethod("IncreaseCapacity", BindingFlags.Instance | BindingFlags.NonPublic)!.Attributes}; nested={typeof(GC.StartNoGCRegionStatus).Attributes}");
+if (!typeof(RuntimeILGenerator).GetField("m_ILStream", BindingFlags.Instance | BindingFlags.NonPublic)!.IsPrivate ||
+    !typeof(RuntimeILGenerator).GetMethod("IncreaseCapacity", BindingFlags.Instance | BindingFlags.NonPublic)!.IsPrivate ||
+    !typeof(GC.StartNoGCRegionStatus).IsNestedPrivate)
+    throw new Exception("The probe must exercise genuinely private members/types in the installed runtime");
+#if MISSING_MEMBER
+Console.WriteLine(il.FieldThatDoesNotExist);
+#endif
+#if WRONG_SIGNATURE
+string wrong = il.m_localCount;
+#endif
+var oldSize = il.m_ILStream.Length;
+il.IncreaseCapacity(4096); // Private method, not just an internal member.
+if (il.m_ILStream.Length <= oldSize) throw new Exception("Private method did not grow the stream");
+il.m_localSignature.AddArgument(typeof(int), false);
+++il.m_localCount;
+var scope = ((DynamicILGenerator)il).m_scope;
+scope.m_tokens.Add(typeof(Math).GetMethod(nameof(Math.Abs), new[] { typeof(int) })!.MethodHandle);
+var token = (scope.m_tokens.Count - 1) | 0x06000000;
+il.m_ILStream[il.m_length++] = (byte)OpCodes.Ldc_I4_S.Value;
+il.m_ILStream[il.m_length++] = unchecked((byte)-42);
+il.UpdateStackSize(OpCodes.Ldc_I4_S, 1);
+il.m_ILStream[il.m_length++] = (byte)OpCodes.Call.Value;
+for (var shift = 0; shift != 32; shift += 8)
+    il.m_ILStream[il.m_length++] = (byte)(token >> shift);
+il.UpdateStackSize(OpCodes.Call, 0);
+il.Emit(OpCodes.Stloc_0);
+il.Emit(OpCodes.Ldloc_0);
+il.Emit(OpCodes.Ret);
+var result = ((Func<int>)method.CreateDelegate(typeof(Func<int>)))();
+if (result != 42) throw new Exception("Unexpected result: " + result);
+Console.WriteLine("Direct private fields + private method + internal types: result=" + result);
+Console.WriteLine("Private nested type: " + typeof(GC.StartNoGCRegionStatus).FullName);
+
+#if !GENERATED_ACCESS_ATTRIBUTE
+namespace System.Runtime.CompilerServices
+{
+    [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true)]
+    internal sealed class IgnoresAccessChecksToAttribute(string assemblyName) : Attribute
+    {
+        public string AssemblyName { get; } = assemblyName;
+    }
+}
+#endif

BuildScripts/IgnoresAccessChecksToGenerator/Directory.Build.props

@@ -0,0 +1,5 @@
+<Project>
+  <PropertyGroup>
+    <LangVersion>latest</LangVersion>
+  </PropertyGroup>
+</Project>

BuildScripts/IgnoresAccessChecksToGenerator/FecSmoke/FecSmoke.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFrameworks>net8.0;net10.0</TargetFrameworks>
+    <OutputType>Exe</OutputType>
+    <RollForward>Disable</RollForward>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>disable</Nullable>
+    <DefineConstants Condition="'$(Light)' == 'true'">$(DefineConstants);LIGHT_EXPRESSION</DefineConstants>
+    <FacadeDirectory Condition="'$(FacadeDirectory)' == ''">../artifacts/$(TargetFramework)/private</FacadeDirectory>
+  </PropertyGroup>
+  <ItemGroup>
+    <AssemblyAttribute Include="System.Runtime.Versioning.SupportedOSPlatformAttribute" Condition="'$(OS)' == 'Windows_NT'">
+      <_Parameter1>windows</_Parameter1>
+    </AssemblyAttribute>
+    <Compile Include="../artifacts/Compiler.Direct.cs" />
+    <Compile Include="../../../src/FastExpressionCompiler/ILReader.cs" />
+    <Compile Include="../../../src/FastExpressionCompiler/ImTools.cs" />
+    <Compile Include="../../../src/FastExpressionCompiler.LightExpression/*.cs" Condition="'$(Light)' == 'true'" />
+    <Compile Include="../../../src/FastExpressionCompiler/TestTools.cs" />
+  </ItemGroup>
+  <Import Project="../build/RuntimeReferences.targets" />
+</Project>

BuildScripts/IgnoresAccessChecksToGenerator/FecSmoke/Program.cs

@@ -0,0 +1,42 @@
+using System.Runtime.CompilerServices;
+#if LIGHT_EXPRESSION
+using FastExpressionCompiler.LightExpression;
+using FastExpressionCompiler.LightExpression.UnitTests;
+using E = FastExpressionCompiler.LightExpression.Expression;
+#else
+using FastExpressionCompiler;
+using FastExpressionCompiler.UnitTests;
+using E = System.Linq.Expressions.Expression;
+#endif
+
+[assembly: IgnoresAccessChecksTo("System.Private.CoreLib")]
+
+var expected = Environment.GetEnvironmentVariable("EXPECTED_CORELIB");
+if (expected != null && !string.Equals(Path.GetFullPath(typeof(object).Assembly.Location), Path.GetFullPath(expected), StringComparison.OrdinalIgnoreCase))
+    throw new Exception("FEC smoke executed a different runtime than the selected reference");
+var parameter = E.Parameter(typeof(int), "value");
+var local = E.Variable(typeof(int), "local");
+var block = E.Block(new[] { local },
+    E.Assign(local, E.Add(parameter, E.Constant(1))),
+    E.TryFinally(E.Add(local, E.Constant(1)), E.Empty()));
+var lambda = E.Lambda<Func<int, int>>(block, parameter);
+for (var i = 0; i < 100; ++i)
+{
+    var compiled = lambda.CompileFast(true);
+    if (compiled(40) != 42) throw new Exception("Compilation/local/exception smoke failed");
+}
+var nested = E.Lambda<Func<int, Func<int>>>(E.Lambda<Func<int>>(E.Add(parameter, E.Constant(2))), parameter).CompileFast(true);
+if (nested(40)() != 42 || nested(50)() != 52) throw new Exception("Nested closure smoke failed");
+var tests = new TestRun(TestFlags.RethrowException);
+tests.Run(new RuntimeAccessorContractTests());
+if (tests.FailedTestCount != 0) throw new Exception("Runtime contracts failed");
+Console.WriteLine($"{typeof(ExpressionCompiler).Namespace}: 100 repeated local/exception compilations, nested capture and {tests.TotalTestCount} runtime contracts passed");
+
+namespace System.Runtime.CompilerServices
+{
+    [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true)]
+    internal sealed class IgnoresAccessChecksToAttribute(string assemblyName) : Attribute
+    {
+        public string AssemblyName { get; } = assemblyName;
+    }
+}

BuildScripts/IgnoresAccessChecksToGenerator/FecSmoke/RuntimeAccessorContractTests.cs

@@ -0,0 +1,159 @@
+#if NET8_0_OR_GREATER
+using System;
+using System.Collections.Generic;
+using System.Reflection;
+using System.Reflection.Emit;
+#if NET10_0_OR_GREATER
+using System.Runtime.CompilerServices;
+#endif
+
+#if LIGHT_EXPRESSION
+namespace FastExpressionCompiler.LightExpression.UnitTests;
+#else
+namespace FastExpressionCompiler.UnitTests;
+#endif
+
+// These are intentionally strict contracts for the CoreCLR versions exercised by CI.
+// A runtime update must not silently disable an optimization while correctness tests pass.
+public struct RuntimeAccessorContractTests : ITestX
+{
+    private const BindingFlags NonPublicStatic = BindingFlags.NonPublic | BindingFlags.Static;
+    private const BindingFlags InstanceMembers = BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public;
+
+    public void Run(TestRun t)
+    {
+        Local_allocation_and_generator_reuse_remain_available(t);
+#if NET10_0_OR_GREATER
+        Accessor_declarations_match_the_runtime_members(t);
+        Production_accessors_bind_and_emit_a_working_method(t);
+#endif
+    }
+
+    public void Local_allocation_and_generator_reuse_remain_available(TestContext t)
+    {
+        var hacks = typeof(DynamicMethodHacks);
+        t.IsNotNull(hacks.GetField("GetNextLocalVarLocation", NonPublicStatic).GetValue(null));
+        t.IsNotNull(hacks.GetField("ReuseDynamicILGenerator", NonPublicStatic).GetValue(null));
+        var pool = hacks.GetField("_pooledILGenerator", NonPublicStatic);
+        var previous = pool.GetValue(null);
+        pool.SetValue(null, null);
+        try
+        {
+            var first = new DynamicMethod("first", typeof(int), new[] { typeof(int) }, typeof(RuntimeAccessorContractTests), true);
+            var il = DynamicMethodHacks.RentPooledOrNewILGenerator(first, typeof(int), new[] { typeof(int) });
+            t.AreEqual(0, il.GetNextLocalVarIndex(typeof(int)));
+            il.BeginExceptionBlock();
+            il.Emit(OpCodes.Ldarg_0);
+            il.Emit(OpCodes.Call, typeof(Math).GetMethod(nameof(Math.Abs), new[] { typeof(int) }));
+            il.Emit(OpCodes.Stloc_0);
+            il.BeginFinallyBlock();
+            il.Emit(OpCodes.Nop);
+            il.EndExceptionBlock();
+            il.Emit(OpCodes.Ldloc_0);
+            il.Emit(OpCodes.Ret);
+            var f = (Func<int, int>)first.CreateDelegate(typeof(Func<int, int>));
+            t.AreEqual(42, f(-42));
+            DynamicMethodHacks.FreePooledILGenerator(first, il);
+
+            // Change the signature and local type after tokens, exception regions and locals were used.
+            var second = new DynamicMethod("second", typeof(string), new[] { typeof(string) }, typeof(RuntimeAccessorContractTests), true);
+            var reused = DynamicMethodHacks.RentPooledOrNewILGenerator(second, typeof(string), new[] { typeof(string) });
+            t.IsTrue(ReferenceEquals(il, reused));
+            t.AreEqual(0, reused.GetNextLocalVarIndex(typeof(string)));
+            reused.Emit(OpCodes.Ldarg_0);
+            reused.Emit(OpCodes.Stloc_0);
+            reused.Emit(OpCodes.Ldloc_0);
+            reused.Emit(OpCodes.Ret);
+            var g = (Func<string, string>)second.CreateDelegate(typeof(Func<string, string>));
+            t.AreEqual("second", g("second"));
+            t.AreEqual(17, f(-17)); // Reuse must not change the already-created method.
+            DynamicMethodHacks.FreePooledILGenerator(second, reused);
+        }
+        finally
+        {
+            pool.SetValue(null, previous);
+        }
+    }
+
+#if NET10_0_OR_GREATER
+    public void Accessor_declarations_match_the_runtime_members(TestContext t)
+    {
+        var count = 0;
+        foreach (var accessor in typeof(DynamicMethodHacks).GetMethods(NonPublicStatic))
+        {
+            var attribute = accessor.GetCustomAttribute<UnsafeAccessorAttribute>();
+            if (attribute == null)
+                continue;
+            ++count;
+            var parameters = accessor.GetParameters();
+            var ownerName = parameters[0].GetCustomAttribute<UnsafeAccessorTypeAttribute>().TypeName;
+            var owner = typeof(ILGenerator).Assembly.GetType(ownerName, throwOnError: true);
+            // UnsafeAccessor does not search base classes. Check the exact declaring type.
+            if (attribute.Kind == UnsafeAccessorKind.Field)
+            {
+                var field = owner.GetField(attribute.Name, InstanceMembers | BindingFlags.DeclaredOnly);
+                if (field == null)
+                    throw new MissingFieldException(owner.FullName, attribute.Name);
+                t.IsTrue(!field.IsStatic);
+                t.AreEqual(field.FieldType.MakeByRefType(), accessor.ReturnType);
+                t.AreEqual(1, parameters.Length);
+            }
+            else
+            {
+                t.AreEqual(UnsafeAccessorKind.Method, attribute.Kind);
+                var argumentTypes = new Type[parameters.Length - 1];
+                for (var i = 1; i < parameters.Length; ++i)
+                    argumentTypes[i - 1] = parameters[i].ParameterType;
+                var method = owner.GetMethod(attribute.Name, InstanceMembers | BindingFlags.DeclaredOnly, null, argumentTypes, null);
+                if (method == null)
+                    throw new MissingMethodException(owner.FullName, attribute.Name);
+                t.IsTrue(!method.IsStatic);
+                t.AreEqual(method.ReturnType, accessor.ReturnType);
+            }
+        }
+        t.AreEqual(6, count);
+    }
+
+    private delegate ref T FieldAccessor<T>(object instance);
+
+    private static T Bind<T>(string name) where T : Delegate =>
+        (T)typeof(DynamicMethodHacks).GetMethod(name, NonPublicStatic).CreateDelegate(typeof(T));
+
+    public void Production_accessors_bind_and_emit_a_working_method(TestContext t)
+    {
+        // Bind the production declarations, not duplicate UnsafeAccessor stubs in the test assembly.
+        var signature = Bind<FieldAccessor<SignatureHelper>>("GetMLocalSignature");
+        var localCount = Bind<FieldAccessor<int>>("GetMLocalCount");
+        var length = Bind<FieldAccessor<int>>("GetMLength");
+        var stream = Bind<FieldAccessor<byte[]>>("GetMILStream");
+        var tokens = Bind<FieldAccessor<List<object>>>("GetMTokens");
+        var updateStack = Bind<Action<object, OpCode, int>>("UpdateStackSize");
+        var method = new DynamicMethod("accessors", typeof(int), Type.EmptyTypes, typeof(RuntimeAccessorContractTests), true);
+        var il = method.GetILGenerator(32);
+        t.AreEqual(0, localCount(il));
+        signature(il).AddArgument(typeof(int), false);
+        ++localCount(il);
+        t.AreEqual(1, localCount(il));
+
+        var scope = il.GetType().GetField("m_scope", InstanceMembers).GetValue(il);
+        var table = tokens(scope);
+        table.Add(typeof(Math).GetMethod(nameof(Math.Abs), new[] { typeof(int) }).MethodHandle);
+        var token = (table.Count - 1) | 0x06000000;
+        var bytes = stream(il);
+        ref var position = ref length(il);
+        t.AreEqual(0, position);
+        bytes[position++] = (byte)OpCodes.Ldc_I4_S.Value;
+        bytes[position++] = unchecked((byte)-42);
+        updateStack(il, OpCodes.Ldc_I4_S, 1);
+        bytes[position++] = (byte)OpCodes.Call.Value;
+        for (var shift = 0; shift != 32; shift += 8)
+            bytes[position++] = (byte)(token >> shift);
+        updateStack(il, OpCodes.Call, 0); // Abs consumes and returns one value.
+        il.Emit(OpCodes.Stloc_0);
+        il.Emit(OpCodes.Ldloc_0);
+        il.Emit(OpCodes.Ret);
+        t.AreEqual(42, ((Func<int>)method.CreateDelegate(typeof(Func<int>)))());
+    }
+#endif
+}
+#endif

BuildScripts/IgnoresAccessChecksToGenerator/Generator/Generator.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <OutputType>Exe</OutputType>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="Microsoft.Build.Framework" HintPath="$(MSBuildBinPath)/Microsoft.Build.Framework.dll" Private="false" />
+    <Reference Include="Microsoft.Build.Utilities.Core" HintPath="$(MSBuildBinPath)/Microsoft.Build.Utilities.Core.dll" Private="false" />
+  </ItemGroup>
+</Project>