diff --git a/.github/workflows/performance-comment.yml b/.github/workflows/performance-comment.yml new file mode 100644 index 000000000..732753b57 --- /dev/null +++ b/.github/workflows/performance-comment.yml @@ -0,0 +1,46 @@ +name: Publish PR performance + +on: + workflow_run: + workflows: [PR performance] + types: [completed] + +permissions: + contents: read + actions: read + pull-requests: write + +concurrency: + group: performance-comment-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: false + +jobs: + comment: + if: github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + # Only trusted default-branch code. Never check out/build the PR in this job. + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + path: trusted + - name: Resolve current PR and download bounded data + id: prepare + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + script: | + const publisher = require('./trusted/test/FastExpressionCompiler.Benchmarks/Regression/publish.cjs'); + await publisher.prepare({github, context, core, directory: require("node:path").join(process.env.RUNNER_TEMP, "performance-comment")}); + - name: Validate and render data without extracting it + if: steps.prepare.outputs.publish == 'true' + shell: bash + run: python3 trusted/test/FastExpressionCompiler.Benchmarks/Regression/publish.py --directory "$RUNNER_TEMP/performance-comment" + - name: Publish current report and outdate older comments + if: steps.prepare.outputs.publish == 'true' + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + script: | + const publisher = require('./trusted/test/FastExpressionCompiler.Benchmarks/Regression/publish.cjs'); + await publisher.post({github, context, directory: require("node:path").join(process.env.RUNNER_TEMP, "performance-comment")}); diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 000000000..caeff1376 --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,138 @@ +name: PR performance + +on: + pull_request: + branches: [master] + types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: + inputs: + pull_request: + description: Pull request number to compare with its current base + required: true + type: string + runtime: + description: Target runtime + type: choice + options: [net10.0, net8.0, both] + default: net10.0 + +permissions: + contents: read + pull-requests: read + +concurrency: + group: performance-${{ github.event.pull_request.number || inputs.pull_request }} + cancel-in-progress: true + +jobs: + revisions: + runs-on: ubuntu-24.04 + outputs: + number: ${{ steps.pr.outputs.number }} + base: ${{ steps.pr.outputs.base }} + head: ${{ steps.pr.outputs.head }} + head-repo: ${{ steps.pr.outputs.head-repo }} + runtimes: ${{ steps.pr.outputs.runtimes }} + steps: + - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + id: pr + env: + REQUESTED_PR: ${{ inputs.pull_request }} + REQUESTED_RUNTIME: ${{ inputs.runtime }} + with: + script: | + const text = process.env.REQUESTED_PR; + if (context.eventName === 'workflow_dispatch' && !/^[1-9][0-9]*$/.test(text)) throw new Error('Invalid PR number'); + const pr = context.eventName === 'pull_request' ? context.payload.pull_request : + (await github.rest.pulls.get({...context.repo, pull_number: Number(text)})).data; + if (pr.base.ref !== 'master' || pr.base.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`) throw new Error('Expected a PR targeting master in this repository'); + core.setOutput('number', pr.number); + core.setOutput('base', pr.base.sha); + core.setOutput('head', pr.head.sha); + core.setOutput('head-repo', pr.head.repo.full_name); + const runtime = process.env.REQUESTED_RUNTIME || 'net10.0'; + if (!['net10.0', 'net8.0', 'both'].includes(runtime)) throw new Error('Invalid runtime'); + core.setOutput('runtimes', JSON.stringify(runtime === 'both' ? ['net8.0', 'net10.0'] : [runtime])); + + benchmark: + needs: revisions + runs-on: ubuntu-24.04 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + runtime: ${{ fromJSON(needs.revisions.outputs.runtimes) }} + env: + PR_NUMBER: ${{ needs.revisions.outputs.number }} + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + DOTNET_NOLOGO: 1 + DOTNET_TieredCompilation: 0 + DOTNET_ReadyToRun: 0 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ needs.revisions.outputs.base }} + path: baseline + persist-credentials: false + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: ${{ needs.revisions.outputs.head-repo }} + ref: ${{ needs.revisions.outputs.head }} + path: candidate + persist-credentials: false + - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4 + with: + dotnet-version: 10.0.301 + # The extended .NET 8 pass needs the runtime as well as the pinned SDK. + - if: matrix.runtime == 'net8.0' + uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4 + with: + dotnet-version: 8.0.422 + - name: Select one frozen harness + shell: bash + run: | + suite=test/FastExpressionCompiler.Benchmarks/Regression + harness="baseline/$suite" + if [ ! -f "$harness/run.py" ]; then harness="candidate/$suite"; fi + echo "PERFORMANCE_HARNESS=$harness" >> "$GITHUB_ENV" + - name: Harness and policy tests + shell: bash + run: | + python3 -m unittest discover -s "$PERFORMANCE_HARNESS/tests" -v + node --test "$PERFORMANCE_HARNESS/tests/publish.test.cjs" + - name: Benchmark exact base and PR head in both orders + id: measure + continue-on-error: true + env: + RUNTIME: ${{ matrix.runtime }} + shell: bash + run: python3 "$PERFORMANCE_HARNESS/run.py" --baseline baseline --candidate candidate --runtime "$RUNTIME" --output results + - name: Compare (reporting only) + if: always() && steps.measure.outcome != 'skipped' + shell: bash + run: | + status=0 + python3 "$PERFORMANCE_HARNESS/compare.py" --input results || status=$? + if [ -f results/report.md ]; then cat results/report.md >> "$GITHUB_STEP_SUMMARY"; fi + exit "$status" + - name: Upload comment data + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: performance-report-${{ matrix.runtime }}-${{ github.run_attempt }} + path: results/report.json + if-no-files-found: warn + retention-days: 14 + - name: Upload complete benchmark evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: performance-data-${{ matrix.runtime }}-${{ github.run_attempt }} + path: | + results/context.json + results/sources.json + results/report.md + results/logs + results/runs + if-no-files-found: warn + retention-days: 14 diff --git a/test/FastExpressionCompiler.Benchmarks/FastExpressionCompiler.Benchmarks.csproj b/test/FastExpressionCompiler.Benchmarks/FastExpressionCompiler.Benchmarks.csproj index ea7f8a3e8..300650b53 100644 --- a/test/FastExpressionCompiler.Benchmarks/FastExpressionCompiler.Benchmarks.csproj +++ b/test/FastExpressionCompiler.Benchmarks/FastExpressionCompiler.Benchmarks.csproj @@ -17,6 +17,7 @@ + diff --git a/test/FastExpressionCompiler.Benchmarks/Regression/.gitignore b/test/FastExpressionCompiler.Benchmarks/Regression/.gitignore new file mode 100644 index 000000000..dfc3a800c --- /dev/null +++ b/test/FastExpressionCompiler.Benchmarks/Regression/.gitignore @@ -0,0 +1,5 @@ +/compiler/ +/bin/ +/obj/ +__pycache__/ +/BenchmarkDotNet.Artifacts/ diff --git a/test/FastExpressionCompiler.Benchmarks/Regression/Adapter.Light.csproj b/test/FastExpressionCompiler.Benchmarks/Regression/Adapter.Light.csproj new file mode 100644 index 000000000..d7ba19a44 --- /dev/null +++ b/test/FastExpressionCompiler.Benchmarks/Regression/Adapter.Light.csproj @@ -0,0 +1,4 @@ + + net8.0;net10.0Regression.LightLIGHT_EXPRESSION + compiler/FastExpressionCompiler.LightExpression.dll + diff --git a/test/FastExpressionCompiler.Benchmarks/Regression/Adapter.Standard.csproj b/test/FastExpressionCompiler.Benchmarks/Regression/Adapter.Standard.csproj new file mode 100644 index 000000000..ce0795b63 --- /dev/null +++ b/test/FastExpressionCompiler.Benchmarks/Regression/Adapter.Standard.csproj @@ -0,0 +1,4 @@ + + net8.0;net10.0Regression.Standard + compiler/FastExpressionCompiler.dll + diff --git a/test/FastExpressionCompiler.Benchmarks/Regression/Directory.Build.props b/test/FastExpressionCompiler.Benchmarks/Regression/Directory.Build.props new file mode 100644 index 000000000..51451eb4c --- /dev/null +++ b/test/FastExpressionCompiler.Benchmarks/Regression/Directory.Build.props @@ -0,0 +1,11 @@ + + + latest + enable + enable + false + obj/$(MSBuildProjectName)/ + $(BaseIntermediateOutputPath) + false + + diff --git a/test/FastExpressionCompiler.Benchmarks/Regression/Program.cs b/test/FastExpressionCompiler.Benchmarks/Regression/Program.cs new file mode 100644 index 000000000..8dd85ed39 --- /dev/null +++ b/test/FastExpressionCompiler.Benchmarks/Regression/Program.cs @@ -0,0 +1,95 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using Perfolizer.Horology; + +namespace Regression; + +[MemoryDiagnoser] +[JsonExporterAttribute.Full] +public class PerformanceBenchmarks +{ + [Params("Standard", "Light")] public string Backend { get; set; } = "Standard"; + [Params("Scalar", "Arithmetic", "CallConditional", "Closure")] public string Shape { get; set; } = "Scalar"; + private Func compile = null!; + private Func construct = null!; + private Func? scalar; + private Func>? closure; + private Delegate compiled = null!; + + [GlobalSetup] + public void Setup() + { + if (Backend == "Standard") + { + compile = Standard.Workloads.Prepare(Shape); + construct = () => Standard.Workloads.Construct(Shape); + Standard.Workloads.Verify(Shape, compile); + } + else + { + compile = Light.Workloads.Prepare(Shape); + construct = () => Light.Workloads.Construct(Shape); + Light.Workloads.Verify(Shape, compile); + } + compiled = compile(); + if (Shape == "Closure") { closure = (Func>)compiled; _ = closure(3)(); } + else { scalar = (Func)compiled; _ = scalar(3); } + } + + [Benchmark] public object Construct() => construct(); + [Benchmark] public Delegate Compile() => compile(); + [Benchmark] + public int CompileAndFirstInvoke() + { + var fresh = compile(); + return Shape == "Closure" ? ((Func>)fresh)(3)() : ((Func)fresh)(3); + } + + // Amortize harness overhead; return a consumed checksum to keep every call observable. + [Benchmark(OperationsPerInvoke = 256)] + public int Invoke() + { + var sum = 0; + if (Shape == "Closure") + { + var f = closure!; + for (var i = 0; i < 256; ++i) sum += f((i & 15) - 8)(); + } + else + { + var f = scalar!; + for (var i = 0; i < 256; ++i) sum += f((i & 15) - 8); + } + return sum; + } +} + +public static class Program +{ + public static int Main(string[] args) + { + if (args.Contains("--verify")) + { + foreach (var backend in new[] { "Standard", "Light" }) + foreach (var shape in new[] { "Scalar", "Arithmetic", "CallConditional", "Closure" }) + { + var benchmark = new PerformanceBenchmarks { Backend = backend, Shape = shape }; + benchmark.Setup(); + var first = shape switch { "Scalar" or "Closure" => 4, "Arithmetic" => 96, _ => 7 }; + var batched = shape switch { "Scalar" or "Closure" => 128, "Arithmetic" => -4096, _ => 1216 }; + if (benchmark.CompileAndFirstInvoke() != first || benchmark.Invoke() != batched || benchmark.Construct() == null) + throw new InvalidOperationException("Incorrect benchmark driver: " + backend + "/" + shape); + } + Console.WriteLine("All 8 workload/backend behavior checks passed."); + return 0; + } + var smoke = args.Contains("--smoke"); + var job = smoke ? Job.Dry : Job.Default.WithLaunchCount(1).WithWarmupCount(2).WithIterationCount(5).WithIterationTime(TimeInterval.FromMilliseconds(100)); + var config = ManualConfig.Create(DefaultConfig.Instance).AddJob(job.WithId(smoke ? "Smoke" : "Regression")); + var summaries = BenchmarkSwitcher.FromTypes(new[] { typeof(PerformanceBenchmarks) }).Run(args.Where(a => a != "--smoke").ToArray(), config).ToArray(); + return summaries.Length == 0 || summaries.Any(s => s.HasCriticalValidationErrors || s.Reports.Any(r => !r.Success)) ? 1 : 0; + } +} diff --git a/test/FastExpressionCompiler.Benchmarks/Regression/README.md b/test/FastExpressionCompiler.Benchmarks/Regression/README.md new file mode 100644 index 000000000..bea240c99 --- /dev/null +++ b/test/FastExpressionCompiler.Benchmarks/Regression/README.md @@ -0,0 +1,84 @@ +# Pull-request performance comparisons + +Addresses the repeatable before/after comparison requested in [#64](https://github.com/dadhi/FastExpressionCompiler/issues/64). This is the reporting phase: performance differences never block merging by default. Build failures and missing/invalid measurements are inconclusive, not passes. The separate minimal-emitted-IL experiment proposed in that discussion is outside this workflow. + +## What runs + +`.github/workflows/performance.yml` runs on PRs targeting master, including new commits and draft PRs. It snapshots the exact event base and head commits, rather than the synthetic merge commit. Each runtime job builds both libraries in Release on one disposable Ubuntu 24.04 runner, using SDK **10.0.301**, BenchmarkDotNet **0.15.8**, tiering and ReadyToRun disabled. Nothing changes the existing exploratory benchmark entry point. + +The default PR pass targets .NET 10. The manual **PR performance** workflow accepts a PR number and can target .NET 8, .NET 10 or both. The .NET 8 installation is pinned to SDK 8.0.422 (runtime 8.0.28); the actual build SDK remains 10.0.301. Keep these explicit pins current when updating supported targets. + +Four workloads cover scalar arithmetic, a balanced arithmetic tree, method/property access with a condition, and a nested-lambda capture. One `Workloads.cs` is compiled twice, against Standard and LightExpression, so the definitions stay identical. Light construction uses native Light nodes. Strict `CompileFast` rejects fallback; setup checks results for negative, zero and positive inputs. + +Each of the eight workload/backend combinations measures: + +- `Construct`: tree construction, including applicable reflection lookups. +- `Compile`: compile a prebuilt tree and create the delegate. +- `CompileAndFirstInvoke`: compile and invoke each new delegate, including deferred JIT work. Closure factories invoke both the outer and resulting inner delegate. +- `Invoke`: warmed delegate execution in a consumed 256-operation batch. Closure factories include activation and inner invocation. + +MemoryDiagnoser measures managed B/op for every phase. Invocation calls typed compiled delegates directly, with the workload branch outside the batch. Loop/checksum overhead is shared; these are relative regression probes, not minimum delegate-call latency claims. Compilation and first invocation stay separate because dynamic-method hosting can move JIT work across the `CreateDelegate` boundary. + +There are 32 cases, five requested measurement iterations after two warmups per case, and two process rounds in opposite order: **base, PR; PR, base**. Builds, correctness checks and outputs are isolated. SDK/source/library hashes, runtime/environment logs and complete BDN JSON are uploaded. The intended hosted-runner budget is roughly 5-15 minutes; the job has a 30-minute cap. Observe actual CI costs before expanding the suite. + +## Stable workloads + +After installation, the harness and comparator come from the exact base commit, and that one harness is copied for both revisions. A PR cannot simply replace its benchmark definitions for only the candidate run. New harness definitions take effect after merging, for subsequent comparisons. + +For this initial installation PR only, the base lacks a harness. A clearly labelled **bootstrap** uses one candidate harness for both revisions. This still tests the same definitions on both versions, but does not claim base-branch provenance. The artifacts record the harness SHA and hashes. + +Benchmark/build code is executable PR code and can deliberately lie or tamper with measurements. These reports are review aids, not tamper-proof attestations or authorization to merge. Do not promote an unreviewed benchmark artifact into a trusted executable. + +## Interpretation and policy + +`compare.py` matches phase/backend/workload plus BDN full identity, job display/configuration, runtime, architecture, SDK and hardware-intrinsic configuration. All 32 identities must be present in all four runs. Duplicate cases, incompatible jobs, unavailable statistics, nonfinite values or fewer than three usable samples make the comparison inconclusive. A dry smoke run intentionally cannot produce a performance pass. + +The report shows absolute times, median per-round relative time changes, allocations and assessments: + +| Evidence | Default action | +| --- | --- | +| Less than 3% time difference | Report as within variation | +| At least 3% slower without confirmation | Warning / noise | +| More than 5% slower and candidate confidence interval entirely above base in **both** rounds | Potential confirmed regression; report only | +| Zero allocation becomes at least 0.5 B/op in **both** rounds | New-allocation regression; report only | +| Other allocation increases above 0.5 B/op in both rounds | Report allocation increase | +| Missing/invalid/mismatched results | Inconclusive; infrastructure check fails | + +BDN's exported confidence intervals are used without pooling different runs. Five samples per round and shared hardware still have limits; no claim of machine isolation is made. Configure `--warning-percent` and `--regression-percent` locally. `--blocking` opts into exit code 2 for confirmed time or new-zero-path allocation regressions; it is deliberately **not enabled in the workflow**. Exit 1 means inconclusive/infrastructure failure; exit 0 is a completed reporting-only comparison, which can still contain reported regressions. + +Before enabling a required regression check, collect same-source controls over time, rerun suspicious cases and calibrate on stable, preferably dedicated isolated hardware. Scheduling, history storage and automatic confirmation runs are future reliability work. Do not run arbitrary fork code on a persistent self-hosted runner. + +## PR comments and trust boundary + +The benchmark job uses `pull_request`, read-only repository permissions, no repository secrets, no persistent checkout credentials and no shared caches. GitHub's normal fork-run approval policy still applies. + +A separate `.github/workflows/performance-comment.yml` uses `workflow_run`. It checks out **only the default branch** and never builds the candidate or executes downloaded files. It resolves the PR via GitHub's authenticated commit association (fork runs can have an empty `pull_requests` array), requires the current PR head and expected workflow/attempt, and reads just one size-bounded `report.json` inside the artifact ZIP without extracting it. The trusted renderer validates identities/numbers and ignores artifact-supplied prose. A reported older base must be verifiable as master history; otherwise the comment is inconclusive. + +For a new head commit it posts a new report, marks the previous bot report **Outdated** and collapses its measurements with a link to the new report. Rerunning the same head updates that head's report instead of duplicating it. Stale heads and late older runs cannot overwrite a newer report. Missing artifacts produce an inconclusive report. Comments link to the run and complete artifacts (14-day retention); they do not delete historical results. + +The publisher must first be merged into the default branch before GitHub can trigger it. Therefore this installation PR can have action summaries/artifacts before merge, but automatic publishing starts after installation. Manual runs publish job summaries/artifacts only; they do not use an artifact-supplied PR identity to obtain comment privileges. + +See GitHub's [secure use reference](https://docs.github.com/en/actions/reference/security/secure-use) for the privilege separation behind this design. + +## Local validation and replay + +With separate clean checkouts for the exact baseline and candidate: + +```sh +python3 test/FastExpressionCompiler.Benchmarks/Regression/run.py \ + --baseline /path/to/base --candidate /path/to/candidate \ + --runtime net10.0 --output /path/to/new-results +python3 test/FastExpressionCompiler.Benchmarks/Regression/compare.py \ + --input /path/to/new-results +``` + +The output directory must be new. `--verify-only` builds both versions and checks all eight behavior oracles without measuring. `--smoke` exercises BDN's Dry job and exports without treating them as meaningful measurements. `--filter` is useful for diagnostics, but an incomplete subset is inconclusive under the full-suite policy. SDK 10.0.301 must be installed; .NET 8 replay also needs its runtime. `logs/environment.log` records the actual environment. + +Run policy/publisher tests with: + +```sh +python3 -m unittest discover -s test/FastExpressionCompiler.Benchmarks/Regression/tests -v +node --test test/FastExpressionCompiler.Benchmarks/Regression/tests/publish.test.cjs +``` + +The tests cover confidence in both rounds, allocation confirmation, reporting versus blocking exit codes, mismatched configurations, missing data, malformed/untrusted ZIPs, stale/foreign reports, fork PR resolution, comment supersession and late-run ordering. They mock GitHub writes; local tests do not post comments. diff --git a/test/FastExpressionCompiler.Benchmarks/Regression/Regression.csproj b/test/FastExpressionCompiler.Benchmarks/Regression/Regression.csproj new file mode 100644 index 000000000..107af2eec --- /dev/null +++ b/test/FastExpressionCompiler.Benchmarks/Regression/Regression.csproj @@ -0,0 +1,9 @@ + + Exenet8.0;net10.0false + + + + + + + diff --git a/test/FastExpressionCompiler.Benchmarks/Regression/Workloads.cs b/test/FastExpressionCompiler.Benchmarks/Regression/Workloads.cs new file mode 100644 index 000000000..c7fc4dea8 --- /dev/null +++ b/test/FastExpressionCompiler.Benchmarks/Regression/Workloads.cs @@ -0,0 +1,62 @@ +#if LIGHT_EXPRESSION +using FastExpressionCompiler.LightExpression; +using E = FastExpressionCompiler.LightExpression.Expression; +using L = FastExpressionCompiler.LightExpression.LambdaExpression; +namespace Regression.Light; +#else +using FastExpressionCompiler; +using E = System.Linq.Expressions.Expression; +using L = System.Linq.Expressions.LambdaExpression; +namespace Regression.Standard; +#endif + +// Compiled twice against the respective compiler binaries: identical workload definitions. +public static class Workloads +{ + public sealed class Model { public int Value { get; set; } = 7; public int Add(int value) => Value + value; } + + private static E Arithmetic(E x, int depth) => depth == 0 ? x : E.Add(Arithmetic(x, depth - 1), Arithmetic(x, depth - 1)); + + private static L Build(string shape) + { + var x = E.Parameter(typeof(int), "x"); + E body = shape switch + { + "Scalar" => E.Add(x, E.Constant(1)), + "Arithmetic" => Arithmetic(x, 5), + "CallConditional" => E.Condition(E.LessThan(x, E.Constant(0)), + E.Call(E.Constant(new Model()), typeof(Model).GetMethod(nameof(Model.Add))!, x), + E.Property(E.Constant(new Model()), nameof(Model.Value))), + "Closure" => E.Lambda>(E.Add(x, E.Constant(1))), + _ => throw new ArgumentOutOfRangeException(nameof(shape)) + }; + return shape == "Closure" ? E.Lambda>>(body, x) : E.Lambda>(body, x); + } + + public static object Construct(string shape) => Build(shape); + + public static Func Prepare(string shape) + { + var tree = Build(shape); + return () => tree.CompileFast(ifFastFailedReturnNull: true) + ?? throw new InvalidOperationException("CompileFast returned null: " + shape); + } + + public static int Invoke(Delegate compiled, int input) => compiled switch + { + Func simple => simple(input), + Func> closure => closure(input)(), + _ => throw new InvalidOperationException("Unexpected delegate type") + }; + + public static void Verify(string shape, Func compile) + { + var compiled = compile(); + foreach (var input in new[] { -2, 0, 3 }) + { + var expected = shape switch { "Scalar" or "Closure" => input + 1, "Arithmetic" => input * 32, + "CallConditional" => input < 0 ? 7 + input : 7, _ => throw new ArgumentException(shape) }; + if (Invoke(compiled, input) != expected) throw new InvalidOperationException("Incorrect result: " + shape); + } + } +} diff --git a/test/FastExpressionCompiler.Benchmarks/Regression/compare.py b/test/FastExpressionCompiler.Benchmarks/Regression/compare.py new file mode 100644 index 000000000..d258f16a9 --- /dev/null +++ b/test/FastExpressionCompiler.Benchmarks/Regression/compare.py @@ -0,0 +1,153 @@ +"""Compare full BenchmarkDotNet JSON; default policy is reporting only.""" +import argparse +import itertools +import json +import math +from pathlib import Path +import re +import statistics + +METHODS = ("Construct", "Compile", "CompileAndFirstInvoke", "Invoke") +BACKENDS = ("Standard", "Light") +SHAPES = ("Scalar", "Arithmetic", "CallConditional", "Closure") +EXPECTED = set(itertools.product(METHODS, BACKENDS, SHAPES)) + + +def finite(value, minimum=None): + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or abs(value) > 1e15: + raise ValueError("Invalid numeric measurement") + if minimum is not None and value < minimum: + raise ValueError("Negative measurement") + return value + + +def validate_stats(s): + if set(s) != {"mean", "lower", "upper", "n", "bytes"}: + raise ValueError("Invalid measurement fields") + for key in ("mean", "lower", "upper", "bytes"): + finite(s[key], 0 if key in ("mean", "bytes") else None) + if s["mean"] <= 0 or not s["lower"] <= s["mean"] <= s["upper"]: + raise ValueError("Invalid confidence interval") + if type(s["n"]) is not int or not 3 <= s["n"] <= 1000: + raise ValueError("Insufficient measurement samples") + + +def validate_report(report): + if not isinstance(report, dict) or not isinstance(report.get("context"), dict) or not isinstance(report.get("rows"), list) or len(report["rows"]) > len(EXPECTED): + raise ValueError("Invalid report shape") + if report.get("schema") != 1 or type(report.get("problems")) is not list or len(report["problems"]) > 128: + raise ValueError("Invalid report schema") + c = report["context"] + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", c["repository"]): + raise ValueError("Invalid repository") + for key in ("base_sha", "head_sha", "harness_sha"): + if not re.fullmatch(r"[0-9a-f]{40}", c[key]): raise ValueError("Invalid revision") + if c["runtime"] not in ("net8.0", "net10.0") or c["sdk"] != "10.0.301": raise ValueError("Invalid runtime") + if c["harness_source"] not in ("base", "bootstrap"): raise ValueError("Invalid harness provenance") + if c["harness_sha"] != c["base_sha" if c["harness_source"] == "base" else "head_sha"]: raise ValueError("Invalid harness revision") + if type(c["number"]) is not int or c["number"] < 0: raise ValueError("Invalid PR number") + seen = set() + for row in report["rows"]: + key = tuple(row[k] for k in ("method", "backend", "shape")) + if key not in EXPECTED or key in seen: raise ValueError("Invalid/duplicate benchmark identity") + seen.add(key) + if len(row["rounds"]) != 2: raise ValueError("Missing process round") + for rd in row["rounds"]: + if set(rd) != {"baseline", "candidate"}: raise ValueError("Missing revision") + for stats in rd.values(): validate_stats(stats) + return seen == EXPECTED and not report["problems"] + + +def classify(row, warning=3.0, regression=5.0): + rounds = row["rounds"] + delta = statistics.median((r["candidate"]["mean"] / r["baseline"]["mean"] - 1) * 100 for r in rounds) + confirmed = all(r["candidate"]["mean"] > r["baseline"]["mean"] * (1 + regression / 100) + and r["candidate"]["lower"] > r["baseline"]["upper"] for r in rounds) + new_alloc = all(r["baseline"]["bytes"] <= .01 and r["candidate"]["bytes"] >= .5 for r in rounds) + more_alloc = all(r["candidate"]["bytes"] > r["baseline"]["bytes"] + .5 for r in rounds) + improved = delta < -warning and all(r["candidate"]["upper"] < r["baseline"]["lower"] for r in rounds) + status = "Regression (both rounds)" if confirmed else "New allocation (both rounds)" if new_alloc else "Allocation increase" if more_alloc else "Warning / noise" if delta >= warning else "Improvement" if improved else "Within variation" + return delta, status, confirmed or new_alloc + + +def render(report, artifact_url=None, warning=3.0, regression=5.0): + complete = validate_report(report) + c = report["context"] + rows = sorted(report["rows"], key=lambda r: (r["backend"], r["method"], r["shape"])) + confirmed = sum(classify(r, warning, regression)[2] for r in rows) + title = "Inconclusive" if not complete else "Potential regressions detected" if confirmed else "Comparison complete" + lines = [f"### {c['runtime']}: {title} (reporting only)", "", + f"Base `{c['base_sha'][:12]}` → PR `{c['head_sha'][:12]}`; SDK `{c['sdk']}`; harness `{c['harness_sha'][:12]}` ({c['harness_source']}).", ""] + if not complete: lines += ["Missing or invalid benchmark results. This is not evidence of a performance pass.", ""] + if c["harness_source"] == "bootstrap": lines += ["Bootstrap: the base does not contain this harness yet. One candidate harness was used for both revisions; later PRs use the base-branch harness.", ""] + lines += ["| Backend / phase / workload | Base ns | PR ns | Time change | B/op: base → PR | Assessment |", "| --- | ---: | ---: | ---: | ---: | --- |"] + for r in rows: + means = {v: statistics.median(rd[v]["mean"] for rd in r["rounds"]) for v in ("baseline", "candidate")} + allocations = {v: statistics.median(rd[v]["bytes"] for rd in r["rounds"]) for v in means} + delta, status, _ = classify(r, warning, regression) + lines.append(f"| {r['backend']} / {r['method']} / {r['shape']} | {means['baseline']:.2f} | {means['candidate']:.2f} | {delta:+.1f}% | {allocations['baseline']:.2f} → {allocations['candidate']:.2f} | {status} |") + lines += ["", f"Two opposite-order process rounds on one GitHub-hosted runner; BDN confidence intervals must be separated in both rounds to classify a >{regression:g}% time regression. A ≥3% change without that evidence is a warning, not proof. Five measurement iterations/round; hosted-runner noise remains.", "", + "Compile excludes delegate invocation; CompileAndFirstInvoke includes the first call/JIT of a new delegate. Invoke is warmed; closure invocation includes creating and calling the returned delegate. MemoryDiagnoser reports managed B/op. Loop/checksum overhead is shared by both revisions."] + if artifact_url: lines += ["", f"[Workflow run and complete raw artifacts]({artifact_url})"] + return "\n".join(lines) + "\n" + + +def read_bdn(folder): + result = {} + for path in folder.rglob("*-full.json"): + data = json.loads(path.read_text(encoding="utf-8-sig")) + environment = data["HostEnvironmentInfo"] + runtime = environment["RuntimeVersion"] + for b in data["Benchmarks"]: + text = b["FullName"] + " " + b["Parameters"] + backend = re.search(r'Backend\s*[:=]\s*"?(Standard|Light)', text) + shape = re.search(r'Shape\s*[:=]\s*"?(Scalar|Arithmetic|CallConditional|Closure)', text) + key = (b["Method"], backend[1] if backend else None, shape[1] if shape else None) + if key not in EXPECTED or key in result: raise ValueError("Unexpected or duplicate benchmark") + s = b["Statistics"]; ci = s["ConfidenceInterval"] + stats = dict(mean=s["Mean"], lower=ci["Lower"], upper=ci["Upper"], n=s["N"], bytes=b["Memory"]["BytesAllocatedPerOperation"]) + validate_stats(stats) + identity = (b["FullName"], b["DisplayInfo"], runtime, environment["BenchmarkDotNetCaption"], + environment["Architecture"], environment["Configuration"], environment["DotNetCliVersion"], + str(b["HardwareIntrinsics"]), ci["Level"]) + result[key] = (stats, identity) + if not result: raise ValueError("No full BDN results") + return result + + +def compare(folder): + report = dict(schema=1, context=json.loads((folder / "context.json").read_text()), rows=[], problems=[]) + runs = {} + for rd, version in itertools.product(range(2), ("baseline", "candidate")): + try: runs[rd, version] = read_bdn(folder / "runs" / f"{rd}-{version}") + except (ValueError, TypeError, KeyError, OSError): + report["problems"].append("Invalid or missing results") + runs[rd, version] = {} + for key in sorted(EXPECTED): + try: + entries = [runs[rd, v][key] for rd, v in itertools.product(range(2), ("baseline", "candidate"))] + if len({e[1] for e in entries}) != 1: raise ValueError("Different execution configurations") + report["rows"].append(dict(zip(("method", "backend", "shape"), key), rounds=[{v: runs[rd, v][key][0] for v in ("baseline", "candidate")} for rd in range(2)])) + except (ValueError, KeyError): report["problems"].append("Missing or mismatched benchmark identity") + return report + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--input", type=Path, required=True) + p.add_argument("--blocking", action="store_true", help="Opt in only after calibrating on stable hardware") + p.add_argument("--warning-percent", type=float, default=3) + p.add_argument("--regression-percent", type=float, default=5) + a = p.parse_args() + if not 0 < a.warning_percent < a.regression_percent < 100: p.error("Require 0 < warning < regression < 100") + report = compare(a.input) + (a.input / "report.json").write_text(json.dumps(report, allow_nan=False, indent=2), encoding="utf-8") + markdown = render(report, warning=a.warning_percent, regression=a.regression_percent) + (a.input / "report.md").write_text(markdown, encoding="utf-8") + print(markdown) + if not validate_report(report): return 1 + return 2 if a.blocking and any(classify(r, a.warning_percent, a.regression_percent)[2] for r in report["rows"]) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/FastExpressionCompiler.Benchmarks/Regression/publish.cjs b/test/FastExpressionCompiler.Benchmarks/Regression/publish.cjs new file mode 100644 index 000000000..df50ca70d --- /dev/null +++ b/test/FastExpressionCompiler.Benchmarks/Regression/publish.cjs @@ -0,0 +1,61 @@ +// Runs only from the default-branch checkout in the privileged workflow. +const fs = require('node:fs/promises'); +const path = require('node:path'); +const marker = /^/; + +async function prepare({github, context, core, directory}) { + const {owner, repo} = context.repo; + const event = context.payload.workflow_run; + const {data: run} = await github.rest.actions.getWorkflowRun({owner, repo, run_id: event.id}); + if (run.event !== 'pull_request' || run.path !== '.github/workflows/performance.yml' || run.status !== 'completed' || + run.repository.full_name !== `${owner}/${repo}` || run.run_attempt !== event.run_attempt) return; + // Fork runs often have an empty pull_requests array; resolve through GitHub, never artifact metadata. + const linked = await github.paginate(github.rest.repos.listPullRequestsAssociatedWithCommit, {owner, repo, commit_sha: run.head_sha, per_page: 100}); + const matches = linked.filter(pr => pr.state === 'open' && pr.base.ref === 'master' && + pr.base.repo.full_name === `${owner}/${repo}` && pr.head.sha === run.head_sha); + if (matches.length !== 1) return; + const pr = matches[0]; + await fs.mkdir(directory, {recursive: true}); + const metadata = {repository: `${owner}/${repo}`, number: pr.number, head_sha: run.head_sha, + run_id: run.id, attempt: run.run_attempt, run_url: run.html_url}; + await fs.writeFile(path.join(directory, 'metadata.json'), JSON.stringify(metadata)); + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, {owner, repo, run_id: run.id, per_page: 100}); + const selected = artifacts.filter(a => !a.expired && a.name === `performance-report-net10.0-${run.run_attempt}`); + if (selected.length === 1 && selected[0].size_in_bytes <= 1024 * 1024) { + const result = await github.rest.actions.downloadArtifact({owner, repo, artifact_id: selected[0].id, archive_format: 'zip'}); + const buffer = Buffer.from(result.data); + if (buffer.length <= 1024 * 1024) await fs.writeFile(path.join(directory, 'performance-report.zip'), buffer); + } + core.setOutput('publish', 'true'); +} + +async function post({github, context, directory}) { + const {owner, repo} = context.repo; + const m = JSON.parse(await fs.readFile(path.join(directory, 'comment.json'), 'utf8')); + const {data: pr} = await github.rest.pulls.get({owner, repo, pull_number: m.number}); + if (pr.state !== 'open' || pr.base.ref !== 'master' || pr.base.repo.full_name !== `${owner}/${repo}` || pr.head.sha !== m.head_sha) return; + if (m.reported_base_sha && m.reported_base_sha !== pr.base.sha) { + // A base branch can advance during the run. Allow its authenticated history, not a fork baseline. + let valid = false; + try { + const {data: comparison} = await github.rest.repos.compareCommitsWithBasehead({owner, repo, basehead: `${m.reported_base_sha}...${pr.base.sha}`}); + valid = comparison.status === 'ahead' || comparison.status === 'identical'; + } catch { /* Publish an inconclusive result instead of trusting an unrelated baseline. */ } + if (!valid) m.body = `### Performance comparison: inconclusive\n\nThe reported baseline could not be verified against master.\n\n[Workflow run](${m.run_url})\n`; + } + const comments = await github.paginate(github.rest.issues.listComments, {owner, repo, issue_number: pr.number, per_page: 100}); + const ours = comments.filter(c => c.user?.login === 'github-actions[bot]' && c.user?.type === 'Bot' && marker.test(c.body || '')); + if (ours.some(c => { const x = c.body.match(marker); return Number(x[2]) > m.run_id || Number(x[2]) === m.run_id && Number(x[3]) > m.attempt; })) return; + const body = `\n${m.body}`; + const sameHead = ours.find(c => c.body.match(marker)[1] === m.head_sha && !c.body.includes('')); + const latest = sameHead + ? (await github.rest.issues.updateComment({owner, repo, comment_id: sameHead.id, body})).data + : (await github.rest.issues.createComment({owner, repo, issue_number: pr.number, body})).data; + for (const old of ours) { + if (old.id === latest.id || old.body.includes('')) continue; + const header = old.body.match(marker)[0]; + await github.rest.issues.updateComment({owner, repo, comment_id: old.id, + body: `${header}\n\n> **Outdated:** superseded by [the newer performance report](${latest.html_url}).\n\n
Previous measurements\n\n${old.body.slice(header.length).trim()}\n\n
`}); + } +} +module.exports = {prepare, post}; diff --git a/test/FastExpressionCompiler.Benchmarks/Regression/publish.py b/test/FastExpressionCompiler.Benchmarks/Regression/publish.py new file mode 100644 index 000000000..eb3073dd6 --- /dev/null +++ b/test/FastExpressionCompiler.Benchmarks/Regression/publish.py @@ -0,0 +1,42 @@ +"""Render bounded, untrusted measurement JSON; never extract or execute artifacts.""" +import argparse +import json +from pathlib import Path +import zipfile +from compare import render, validate_report + + +def read_artifact(path, metadata): + if path.stat().st_size > 1024 * 1024: raise ValueError("Artifact too large") + with zipfile.ZipFile(path) as z: + entries = z.infolist() + if len(entries) != 1 or entries[0].filename != "report.json" or entries[0].file_size > 256 * 1024: + raise ValueError("Unexpected archive contents") + if (entries[0].external_attr >> 16) & 0o170000 == 0o120000: raise ValueError("Symlink artifact") + report = json.loads(z.read(entries[0])) + validate_report(report) + c = report["context"] + if c["repository"] != metadata["repository"] or c["number"] != metadata["number"] or c["head_sha"] != metadata["head_sha"] or c["runtime"] != "net10.0": + raise ValueError("Artifact is not for this PR head/runtime") + return report + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--directory", type=Path, required=True) + a = p.parse_args() + m = json.loads((a.directory / "metadata.json").read_text()) + url = m["run_url"] + try: + report = read_artifact(a.directory / "performance-report.zip", m) + body = render(report, url) + baseline = report["context"]["base_sha"] + except (OSError, ValueError, KeyError, TypeError, zipfile.BadZipFile, RuntimeError): + body = "### Performance comparison: inconclusive\n\nThe benchmark run did not provide valid, complete report data for this PR head. No performance pass is claimed.\n\n" + f"[Run and artifacts]({url})\n" + baseline = None + payload = dict(m, body=body, reported_base_sha=baseline) + (a.directory / "comment.json").write_text(json.dumps(payload), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/test/FastExpressionCompiler.Benchmarks/Regression/run.py b/test/FastExpressionCompiler.Benchmarks/Regression/run.py new file mode 100644 index 000000000..b1901f038 --- /dev/null +++ b/test/FastExpressionCompiler.Benchmarks/Regression/run.py @@ -0,0 +1,69 @@ +"""Build exact revisions and run one frozen BDN harness in counterbalanced rounds.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess + +SDK = "10.0.301" +SUITE = Path("test/FastExpressionCompiler.Benchmarks/Regression") + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--baseline", type=Path, required=True) + p.add_argument("--candidate", type=Path, required=True) + p.add_argument("--output", type=Path, required=True) + p.add_argument("--runtime", choices=["net8.0", "net10.0"], default="net10.0") + p.add_argument("--verify-only", action="store_true") + p.add_argument("--smoke", action="store_true", help="Dry run, not valid performance evidence") + p.add_argument("--filter", default="*") + a = p.parse_args() + out = a.output.resolve() + out.mkdir(parents=True, exist_ok=False) + (out / "logs").mkdir() + roots = {"baseline": a.baseline.resolve(), "candidate": a.candidate.resolve()} + shas = {v: subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=r, text=True).strip() for v, r in roots.items()} + harness_root = roots["baseline"] if (roots["baseline"] / SUITE / "Program.cs").exists() else roots["candidate"] + context = dict(repository=os.environ.get("GITHUB_REPOSITORY", "local/local"), + number=int(os.environ.get("PR_NUMBER", "0")), base_sha=shas["baseline"], head_sha=shas["candidate"], + harness_sha=shas["baseline"] if harness_root == roots["baseline"] else shas["candidate"], + harness_source="base" if harness_root == roots["baseline"] else "bootstrap", + runtime=a.runtime, sdk=SDK) + (out / "context.json").write_text(json.dumps(context, indent=2), encoding="utf-8") + env = dict(os.environ, DOTNET_TieredCompilation="0", DOTNET_ReadyToRun="0", + DOTNET_CLI_TELEMETRY_OPTOUT="1", DOTNET_NOLOGO="1") + def run(name, command, cwd): + print(name, flush=True) + with (out / "logs" / (name + ".log")).open("w", encoding="utf-8") as log: + subprocess.run([str(x) for x in command], cwd=cwd, env=env, stdout=log, stderr=subprocess.STDOUT, check=True) + binaries = {} + for version, root in roots.items(): + dest = out / "harness" / version + shutil.copytree(harness_root / SUITE, dest, ignore=shutil.ignore_patterns("bin", "obj", "compiler", "__pycache__", "BenchmarkDotNet.Artifacts")) + (dest / "global.json").write_text(json.dumps({"sdk": {"version": SDK, "rollForward": "disable"}}), encoding="utf-8") + compiler = dest / "compiler" + for project in ["FastExpressionCompiler", "FastExpressionCompiler.LightExpression"]: + run("build-" + version + "-" + project, ["dotnet", "build", root / "src" / project, "-c", "Release", "-f", a.runtime, + "-p:TargetFrameworks=" + a.runtime, "--disable-build-servers", "-m:1", "-o", compiler], dest) + run("build-harness-" + version, ["dotnet", "build", "Regression.csproj", "-c", "Release", "-f", a.runtime, "--disable-build-servers", "-m:1"], dest) + binaries[version] = dest / "bin" / "Release" / a.runtime / "Regression.dll" + run("verify-" + version, ["dotnet", binaries[version], "--verify"], dest) + manifest = {str(f.relative_to(out)): hashlib.sha256(f.read_bytes()).hexdigest() + for f in (out / "harness").rglob("*") if f.is_file() and (f.suffix in [".cs", ".csproj", ".props"] or f.name in ["FastExpressionCompiler.dll", "FastExpressionCompiler.LightExpression.dll"])} + (out / "sources.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + run("environment", ["dotnet", "--info"], out / "harness" / "baseline") + if a.verify_only: + return + for round_index, order in enumerate([["baseline", "candidate"], ["candidate", "baseline"]]): + for version in order: + command = ["dotnet", binaries[version], "--filter", a.filter, "--artifacts", out / "runs" / f"{round_index}-{version}"] + if a.smoke: + command.append("--smoke") + run(f"benchmark-{round_index}-{version}", command, out / "harness" / version) + + +if __name__ == "__main__": + main() diff --git a/test/FastExpressionCompiler.Benchmarks/Regression/tests/publish.test.cjs b/test/FastExpressionCompiler.Benchmarks/Regression/tests/publish.test.cjs new file mode 100644 index 000000000..60718d70f --- /dev/null +++ b/test/FastExpressionCompiler.Benchmarks/Regression/tests/publish.test.cjs @@ -0,0 +1,46 @@ +const {test} = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const {prepare, post} = require('../publish.cjs'); +const head='b'.repeat(40), base='a'.repeat(40); +const header=(h,run=100,attempt=1)=>``; +const bot={login:'github-actions[bot]',type:'Bot'}; +async function fixture(t) { + const directory=await fs.mkdtemp(path.join(os.tmpdir(),'fec-publisher-')); + t.after(()=>fs.rm(directory,{recursive:true,force:true})); + const calls=[]; + const pr={number:7,state:'open',head:{sha:head},base:{ref:'master',sha:base,repo:{full_name:'dadhi/FastExpressionCompiler'}}}; + const run={id:101,run_attempt:1,event:'pull_request',status:'completed',path:'.github/workflows/performance.yml',repository:{full_name:'dadhi/FastExpressionCompiler'},head_sha:head,html_url:'https://github.com/dadhi/FastExpressionCompiler/actions/runs/101',pull_requests:[]}; + const comments=[]; + const github={rest:{actions:{getWorkflowRun:async()=>({data:run}),listWorkflowRunArtifacts:'artifacts'},repos:{listPullRequestsAssociatedWithCommit:'linked',compareCommitsWithBasehead:async()=>({data:{status:'diverged'}})},pulls:{get:async()=>({data:pr})},issues:{listComments:'comments',createComment:async x=>{calls.push(['create',x]);return{data:{id:99,html_url:'https://github.com/report/99'}}},updateComment:async x=>{calls.push(['update',x]);return{data:{id:x.comment_id,html_url:'https://github.com/report/'+x.comment_id}}}}},paginate:async endpoint=>endpoint==='linked'?[pr]:endpoint==='comments'?comments:[]}; + const context={repo:{owner:'dadhi',repo:'FastExpressionCompiler'},payload:{workflow_run:{id:101,run_attempt:1}}}; + const core={setOutput:(...x)=>calls.push(['output',...x])}; + const input={repository:'dadhi/FastExpressionCompiler',number:7,head_sha:head,run_id:101,attempt:1,run_url:run.html_url,reported_base_sha:base,body:'safe generated report'}; + await fs.writeFile(path.join(directory,'comment.json'),JSON.stringify(input)); + return {directory,github,context,core,calls,pr,run,comments}; +} +test('fork run with empty pull_requests resolves through authenticated commit association',async t=>{ + const f=await fixture(t);await prepare(f);assert.equal(f.calls[0][0],'output');assert.equal(JSON.parse(await fs.readFile(path.join(f.directory,'metadata.json'))).number,7); +}); +test('stale head ignored before artifact download and before posting',async t=>{ + const f=await fixture(t);f.pr.head.sha='c'.repeat(40);await prepare(f);await post(f);assert.equal(f.calls.length,0); +}); +test('wrong workflow and old attempt ignored',async t=>{ + const f=await fixture(t);f.run.path='.github/workflows/other.yml';await prepare(f);assert.equal(f.calls.length,0); + f.run.path='.github/workflows/performance.yml';f.run.run_attempt=2;await prepare(f);assert.equal(f.calls.length,0); +}); +test('new commit creates a new report and collapses the prior report',async t=>{ + const f=await fixture(t);f.comments.push({id:1,user:bot,body:header('c'.repeat(40))+'\nold'});await post(f); + assert.equal(f.calls[0][0],'create');assert.equal(f.calls[1][0],'update');assert.match(f.calls[1][1].body,/Outdated/);assert.match(f.calls[1][1].body,/
/); +}); +test('same-head rerun updates its report instead of duplicating',async t=>{ + const f=await fixture(t);f.comments.push({id:2,user:bot,body:header(head)+'\nold'});await post(f);assert.equal(f.calls.length,1);assert.equal(f.calls[0][0],'update');assert.equal(f.calls[0][1].comment_id,2); +}); +test('late completion cannot replace a newer run',async t=>{ + const f=await fixture(t);f.comments.push({id:2,user:bot,body:header(head,102)+'\nnewer'});await post(f);assert.equal(f.calls.length,0); +}); +test('unverified baseline produces an inconclusive comment',async t=>{ + const f=await fixture(t);f.pr.base.sha='c'.repeat(40);await post(f);assert.match(f.calls[0][1].body,/inconclusive/);assert.doesNotMatch(f.calls[0][1].body,/safe generated report/); +}); diff --git a/test/FastExpressionCompiler.Benchmarks/Regression/tests/test_policy.py b/test/FastExpressionCompiler.Benchmarks/Regression/tests/test_policy.py new file mode 100644 index 000000000..c79dce675 --- /dev/null +++ b/test/FastExpressionCompiler.Benchmarks/Regression/tests/test_policy.py @@ -0,0 +1,128 @@ +import copy +import importlib.util +import io +import json +from pathlib import Path +import sys +import subprocess +import tempfile +import unittest +import zipfile + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import compare +from publish import read_artifact + + +def stats(mean=100, low=99, high=101, allocated=0): + return dict(mean=mean, lower=low, upper=high, n=5, bytes=allocated) + + +def report(): + return dict(schema=1, context=dict(repository="dadhi/FastExpressionCompiler", number=7, + base_sha="a"*40, head_sha="b"*40, harness_sha="a"*40, harness_source="base", runtime="net10.0", sdk="10.0.301"), + problems=[], rows=[dict(zip(("method", "backend", "shape"), k), rounds=[dict(baseline=stats(), candidate=stats()) for _ in range(2)]) for k in sorted(compare.EXPECTED)]) + + +class PolicyTests(unittest.TestCase): + def test_baseline_control_is_complete(self): + self.assertTrue(compare.validate_report(report())) + self.assertIn("Comparison complete", compare.render(report())) + + def test_significance_required_in_both_rounds(self): + row = report()["rows"][0] + row["rounds"][0]["candidate"] = stats(120,119,121) + self.assertFalse(compare.classify(row)[2]) + row["rounds"][1]["candidate"] = stats(120,119,121) + self.assertTrue(compare.classify(row)[2]) + row["rounds"][1]["candidate"] = stats(120,90,150) + self.assertFalse(compare.classify(row)[2]) + + def test_zero_allocation_needs_both_rounds(self): + row = report()["rows"][0] + row["rounds"][0]["candidate"]["bytes"] = 24 + self.assertFalse(compare.classify(row)[2]) + row["rounds"][1]["candidate"]["bytes"] = 24 + self.assertTrue(compare.classify(row)[2]) + + def test_missing_results_are_inconclusive(self): + r = report(); r["rows"].pop() + self.assertFalse(compare.validate_report(r)) + self.assertIn("Inconclusive", compare.render(r)) + + def test_duplicate_and_nonfinite_results_rejected(self): + r = report(); r["rows"].append(copy.deepcopy(r["rows"][0])) + with self.assertRaises(ValueError): compare.validate_report(r) + r = report(); r["rows"][0]["rounds"][0]["candidate"]["mean"] = float("nan") + with self.assertRaises(ValueError): compare.validate_report(r) + + def test_problems_cannot_inject_markdown(self): + r = report(); r["problems"] = ["@everyone [click](https://attacker.invalid)"] + self.assertNotIn("attacker", compare.render(r)) + self.assertNotIn("@everyone", compare.render(r)) + + def test_dry_measurements_not_accepted(self): + r = report(); r["rows"][0]["rounds"][0]["baseline"]["n"] = 1 + with self.assertRaises(ValueError): compare.validate_report(r) + + def test_configuration_mismatch_and_missing_input(self): + with tempfile.TemporaryDirectory() as folder: + d=Path(folder); (d/"context.json").write_text(json.dumps(report()["context"])) + result=compare.compare(d) + self.assertFalse(compare.validate_report(result)) + self.assertEqual(len(result["rows"]),0) + + +class ComparisonIntegrationTests(unittest.TestCase): + def test_cli_policy_and_configuration_identity(self): + with tempfile.TemporaryDirectory() as folder: + d=Path(folder); (d/"context.json").write_text(json.dumps(report()["context"])) + for rd in range(2): + for version in ("baseline","candidate"): + benchmarks=[] + for i,(method,backend,shape) in enumerate(sorted(compare.EXPECTED)): + mean=120 if version=="candidate" and i==0 else 100 + benchmarks.append(dict(Method=method,FullName=f'Regression.PerformanceBenchmarks.{method}(Backend: "{backend}", Shape: "{shape}")', + Parameters=f"Backend={backend}&Shape={shape}",DisplayInfo=f"{method}: Regression [Backend={backend}, Shape={shape}]",HardwareIntrinsics="test", + Statistics=dict(Mean=mean,N=5,ConfidenceInterval=dict(Lower=mean-1,Upper=mean+1,Level=12)),Memory=dict(BytesAllocatedPerOperation=0))) + data=dict(HostEnvironmentInfo=dict(RuntimeVersion=".NET 10.0.9",BenchmarkDotNetCaption="0.15.8",Architecture="X64",Configuration="Release",DotNetCliVersion="10.0.301"),Benchmarks=benchmarks) + path=d/"runs"/f"{rd}-{version}";path.mkdir(parents=True);(path/"suite-full.json").write_text(json.dumps(data)) + command=[sys.executable,str(Path(compare.__file__)),"--input",str(d)] + self.assertEqual(subprocess.run(command,capture_output=True).returncode,0) + self.assertEqual(subprocess.run(command+["--blocking"],capture_output=True).returncode,2) + self.assertIn("Regression (both rounds)",(d/"report.md").read_text()) + path=d/"runs/1-candidate/suite-full.json";data=json.loads(path.read_text());data["HostEnvironmentInfo"]["RuntimeVersion"]="different runtime";path.write_text(json.dumps(data)) + self.assertEqual(subprocess.run(command,capture_output=True).returncode,1) + self.assertIn("Inconclusive",(d/"report.md").read_text()) + + +class ArtifactTests(unittest.TestCase): + metadata = dict(repository="dadhi/FastExpressionCompiler", number=7, head_sha="b"*40) + + def artifact(self, contents, name="report.json"): + f=tempfile.NamedTemporaryFile(suffix=".zip", delete=False) + p=Path(f.name); f.close(); self.addCleanup(p.unlink) + with zipfile.ZipFile(p,"w") as z:z.writestr(name, json.dumps(contents)) + return p + + def test_valid_artifact(self): + self.assertTrue(compare.validate_report(read_artifact(self.artifact(report()), self.metadata))) + + def test_stale_or_foreign_report(self): + for key,value in [("head_sha","c"*40),("repository","attacker/repo"),("number",8),("runtime","net8.0")]: + r=report();r["context"][key]=value + with self.assertRaises(ValueError):read_artifact(self.artifact(r),self.metadata) + + def test_never_extracts_paths_or_accepts_extra_files(self): + with self.assertRaises(ValueError):read_artifact(self.artifact(report(),"../report.json"),self.metadata) + p=self.artifact(report()) + with zipfile.ZipFile(p,"a") as z:z.writestr("run.py","raise RuntimeError('must never execute')") + with self.assertRaises(ValueError):read_artifact(p,self.metadata) + + def test_compressed_oversize_rejected(self): + p=self.artifact(report()) + with zipfile.ZipFile(p,"w",zipfile.ZIP_DEFLATED) as z:z.writestr("report.json"," "*300000) + with self.assertRaises(ValueError):read_artifact(p,self.metadata) + + +if __name__ == "__main__":unittest.main()