Add reporting-only PR performance comparisons

.github/workflows/performance-comment.yml

@@ -0,0 +1,46 @@
+name: Publish PR performance
+
+on:
+  workflow_run:
+    workflows: [PR performance]
+    types: [completed]
+
+permissions:
+  contents: read
+  actions: read
+  pull-requests: write
+
+concurrency:
+  group: performance-comment-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }}
+  cancel-in-progress: false
+
+jobs:
+  comment:
+    if: github.event.workflow_run.event == 'pull_request'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    steps:
+      # Only trusted default-branch code. Never check out/build the PR in this job.
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          persist-credentials: false
+          path: trusted
+      - name: Resolve current PR and download bounded data
+        id: prepare
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        with:
+          script: |
+            const publisher = require('./trusted/test/FastExpressionCompiler.Benchmarks/Regression/publish.cjs');
+            await publisher.prepare({github, context, core, directory: require("node:path").join(process.env.RUNNER_TEMP, "performance-comment")});
+      - name: Validate and render data without extracting it
+        if: steps.prepare.outputs.publish == 'true'
+        shell: bash
+        run: python3 trusted/test/FastExpressionCompiler.Benchmarks/Regression/publish.py --directory "$RUNNER_TEMP/performance-comment"
+      - name: Publish current report and outdate older comments
+        if: steps.prepare.outputs.publish == 'true'
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        with:
+          script: |
+            const publisher = require('./trusted/test/FastExpressionCompiler.Benchmarks/Regression/publish.cjs');
+            await publisher.post({github, context, directory: require("node:path").join(process.env.RUNNER_TEMP, "performance-comment")});

.github/workflows/performance.yml

@@ -0,0 +1,138 @@
+name: PR performance
+
+on:
+  pull_request:
+    branches: [master]
+    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_dispatch:
+    inputs:
+      pull_request:
+        description: Pull request number to compare with its current base
+        required: true
+        type: string
+      runtime:
+        description: Target runtime
+        type: choice
+        options: [net10.0, net8.0, both]
+        default: net10.0
+
+permissions:
+  contents: read
+  pull-requests: read
+
+concurrency:
+  group: performance-${{ github.event.pull_request.number || inputs.pull_request }}
+  cancel-in-progress: true
+
+jobs:
+  revisions:
+    runs-on: ubuntu-24.04
+    outputs:
+      number: ${{ steps.pr.outputs.number }}
+      base: ${{ steps.pr.outputs.base }}
+      head: ${{ steps.pr.outputs.head }}
+      head-repo: ${{ steps.pr.outputs.head-repo }}
+      runtimes: ${{ steps.pr.outputs.runtimes }}
+    steps:
+      - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        id: pr
+        env:
+          REQUESTED_PR: ${{ inputs.pull_request }}
+          REQUESTED_RUNTIME: ${{ inputs.runtime }}
+        with:
+          script: |
+            const text = process.env.REQUESTED_PR;
+            if (context.eventName === 'workflow_dispatch' && !/^[1-9][0-9]*$/.test(text)) throw new Error('Invalid PR number');
+            const pr = context.eventName === 'pull_request' ? context.payload.pull_request :
+              (await github.rest.pulls.get({...context.repo, pull_number: Number(text)})).data;
+            if (pr.base.ref !== 'master' || pr.base.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`) throw new Error('Expected a PR targeting master in this repository');
+            core.setOutput('number', pr.number);
+            core.setOutput('base', pr.base.sha);
+            core.setOutput('head', pr.head.sha);
+            core.setOutput('head-repo', pr.head.repo.full_name);
+            const runtime = process.env.REQUESTED_RUNTIME || 'net10.0';
+            if (!['net10.0', 'net8.0', 'both'].includes(runtime)) throw new Error('Invalid runtime');
+            core.setOutput('runtimes', JSON.stringify(runtime === 'both' ? ['net8.0', 'net10.0'] : [runtime]));
+
+  benchmark:
+    needs: revisions
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        runtime: ${{ fromJSON(needs.revisions.outputs.runtimes) }}
+    env:
+      PR_NUMBER: ${{ needs.revisions.outputs.number }}
+      DOTNET_CLI_TELEMETRY_OPTOUT: 1
+      DOTNET_NOLOGO: 1
+      DOTNET_TieredCompilation: 0
+      DOTNET_ReadyToRun: 0
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          ref: ${{ needs.revisions.outputs.base }}
+          path: baseline
+          persist-credentials: false
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          repository: ${{ needs.revisions.outputs.head-repo }}
+          ref: ${{ needs.revisions.outputs.head }}
+          path: candidate
+          persist-credentials: false
+      - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4
+        with:
+          dotnet-version: 10.0.301
+          # The extended .NET 8 pass needs the runtime as well as the pinned SDK.
+      - if: matrix.runtime == 'net8.0'
+        uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4
+        with:
+          dotnet-version: 8.0.422
+      - name: Select one frozen harness
+        shell: bash
+        run: |
+          suite=test/FastExpressionCompiler.Benchmarks/Regression
+          harness="baseline/$suite"
+          if [ ! -f "$harness/run.py" ]; then harness="candidate/$suite"; fi
+          echo "PERFORMANCE_HARNESS=$harness" >> "$GITHUB_ENV"
+      - name: Harness and policy tests
+        shell: bash
+        run: |
+          python3 -m unittest discover -s "$PERFORMANCE_HARNESS/tests" -v
+          node --test "$PERFORMANCE_HARNESS/tests/publish.test.cjs"
+      - name: Benchmark exact base and PR head in both orders
+        id: measure
+        continue-on-error: true
+        env:
+          RUNTIME: ${{ matrix.runtime }}
+        shell: bash
+        run: python3 "$PERFORMANCE_HARNESS/run.py" --baseline baseline --candidate candidate --runtime "$RUNTIME" --output results
+      - name: Compare (reporting only)
+        if: always() && steps.measure.outcome != 'skipped'
+        shell: bash
+        run: |
+          status=0
+          python3 "$PERFORMANCE_HARNESS/compare.py" --input results || status=$?
+          if [ -f results/report.md ]; then cat results/report.md >> "$GITHUB_STEP_SUMMARY"; fi
+          exit "$status"
+      - name: Upload comment data
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: performance-report-${{ matrix.runtime }}-${{ github.run_attempt }}
+          path: results/report.json
+          if-no-files-found: warn
+          retention-days: 14
+      - name: Upload complete benchmark evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: performance-data-${{ matrix.runtime }}-${{ github.run_attempt }}
+          path: |
+            results/context.json
+            results/sources.json
+            results/report.md
+            results/logs
+            results/runs
+          if-no-files-found: warn
+          retention-days: 14

test/FastExpressionCompiler.Benchmarks/FastExpressionCompiler.Benchmarks.csproj

@@ -17,6 +17,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <Compile Remove="Regression/**/*.cs" />
         <PackageReference Include="BenchmarkDotNet" Version="0.15.8"/>
         <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.15.8" />
     </ItemGroup>

test/FastExpressionCompiler.Benchmarks/Regression/.gitignore

@@ -0,0 +1,5 @@
+/compiler/
+/bin/
+/obj/
+__pycache__/
+/BenchmarkDotNet.Artifacts/

test/FastExpressionCompiler.Benchmarks/Regression/Adapter.Light.csproj

@@ -0,0 +1,4 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup><TargetFrameworks>net8.0;net10.0</TargetFrameworks><AssemblyName>Regression.Light</AssemblyName><DefineConstants>LIGHT_EXPRESSION</DefineConstants></PropertyGroup>
+  <ItemGroup><Compile Include="Workloads.cs" /><Reference Include="FastExpressionCompiler.LightExpression"><HintPath>compiler/FastExpressionCompiler.LightExpression.dll</HintPath></Reference></ItemGroup>
+</Project>

test/FastExpressionCompiler.Benchmarks/Regression/Adapter.Standard.csproj

@@ -0,0 +1,4 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup><TargetFrameworks>net8.0;net10.0</TargetFrameworks><AssemblyName>Regression.Standard</AssemblyName></PropertyGroup>
+  <ItemGroup><Compile Include="Workloads.cs" /><Reference Include="FastExpressionCompiler"><HintPath>compiler/FastExpressionCompiler.dll</HintPath></Reference></ItemGroup>
+</Project>

test/FastExpressionCompiler.Benchmarks/Regression/Directory.Build.props

@@ -0,0 +1,11 @@
+<Project>
+  <PropertyGroup>
+    <LangVersion>latest</LangVersion>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <EnableDefaultCompileItems>false</EnableDefaultCompileItems>
+    <BaseIntermediateOutputPath>obj/$(MSBuildProjectName)/</BaseIntermediateOutputPath>
+    <MSBuildProjectExtensionsPath>$(BaseIntermediateOutputPath)</MSBuildProjectExtensionsPath>
+    <IsTestProject>false</IsTestProject>
+  </PropertyGroup>
+</Project>

test/FastExpressionCompiler.Benchmarks/Regression/Program.cs

@@ -0,0 +1,95 @@
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Exporters;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Jobs;
+using Perfolizer.Horology;
+
+namespace Regression;
+
+[MemoryDiagnoser]
+[JsonExporterAttribute.Full]
+public class PerformanceBenchmarks
+{
+    [Params("Standard", "Light")] public string Backend { get; set; } = "Standard";
+    [Params("Scalar", "Arithmetic", "CallConditional", "Closure")] public string Shape { get; set; } = "Scalar";
+    private Func<Delegate> compile = null!;
+    private Func<object> construct = null!;
+    private Func<int, int>? scalar;
+    private Func<int, Func<int>>? closure;
+    private Delegate compiled = null!;
+
+    [GlobalSetup]
+    public void Setup()
+    {
+        if (Backend == "Standard")
+        {
+            compile = Standard.Workloads.Prepare(Shape);
+            construct = () => Standard.Workloads.Construct(Shape);
+            Standard.Workloads.Verify(Shape, compile);
+        }
+        else
+        {
+            compile = Light.Workloads.Prepare(Shape);
+            construct = () => Light.Workloads.Construct(Shape);
+            Light.Workloads.Verify(Shape, compile);
+        }
+        compiled = compile();
+        if (Shape == "Closure") { closure = (Func<int, Func<int>>)compiled; _ = closure(3)(); }
+        else { scalar = (Func<int, int>)compiled; _ = scalar(3); }
+    }
+
+    [Benchmark] public object Construct() => construct();
+    [Benchmark] public Delegate Compile() => compile();
+    [Benchmark]
+    public int CompileAndFirstInvoke()
+    {
+        var fresh = compile();
+        return Shape == "Closure" ? ((Func<int, Func<int>>)fresh)(3)() : ((Func<int, int>)fresh)(3);
+    }
+
+    // Amortize harness overhead; return a consumed checksum to keep every call observable.
+    [Benchmark(OperationsPerInvoke = 256)]
+    public int Invoke()
+    {
+        var sum = 0;
+        if (Shape == "Closure")
+        {
+            var f = closure!;
+            for (var i = 0; i < 256; ++i) sum += f((i & 15) - 8)();
+        }
+        else
+        {
+            var f = scalar!;
+            for (var i = 0; i < 256; ++i) sum += f((i & 15) - 8);
+        }
+        return sum;
+    }
+}
+
+public static class Program
+{
+    public static int Main(string[] args)
+    {
+        if (args.Contains("--verify"))
+        {
+            foreach (var backend in new[] { "Standard", "Light" })
+            foreach (var shape in new[] { "Scalar", "Arithmetic", "CallConditional", "Closure" })
+            {
+                var benchmark = new PerformanceBenchmarks { Backend = backend, Shape = shape };
+                benchmark.Setup();
+                var first = shape switch { "Scalar" or "Closure" => 4, "Arithmetic" => 96, _ => 7 };
+                var batched = shape switch { "Scalar" or "Closure" => 128, "Arithmetic" => -4096, _ => 1216 };
+                if (benchmark.CompileAndFirstInvoke() != first || benchmark.Invoke() != batched || benchmark.Construct() == null)
+                    throw new InvalidOperationException("Incorrect benchmark driver: " + backend + "/" + shape);
+            }
+            Console.WriteLine("All 8 workload/backend behavior checks passed.");
+            return 0;
+        }
+        var smoke = args.Contains("--smoke");
+        var job = smoke ? Job.Dry : Job.Default.WithLaunchCount(1).WithWarmupCount(2).WithIterationCount(5).WithIterationTime(TimeInterval.FromMilliseconds(100));
+        var config = ManualConfig.Create(DefaultConfig.Instance).AddJob(job.WithId(smoke ? "Smoke" : "Regression"));
+        var summaries = BenchmarkSwitcher.FromTypes(new[] { typeof(PerformanceBenchmarks) }).Run(args.Where(a => a != "--smoke").ToArray(), config).ToArray();
+        return summaries.Length == 0 || summaries.Any(s => s.HasCriticalValidationErrors || s.Reports.Any(r => !r.Success)) ? 1 : 0;
+    }
+}