Skip to content

Commit 280cbaf

Browse files
dadhidadhi
authored
Merge pull request #564 from jogibear9988/ci/pr-performance-reports
Add reporting-only PR performance comparisons
2 parents df1648a + ff47206 commit 280cbaf

17 files changed

Lines changed: 958 additions & 0 deletions

‎.github/workflows/performance-comment.yml‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
name: Publish PR performance
2+
3+
on:
4+
workflow_run:
5+
workflows: [PR performance]
6+
types: [completed]
7+
8+
permissions:
9+
contents: read
10+
actions: read
11+
pull-requests: write
12+
13+
concurrency:
14+
group: performance-comment-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }}
15+
cancel-in-progress: false
16+
17+
jobs:
18+
comment:
19+
if: github.event.workflow_run.event == 'pull_request'
20+
runs-on: ubuntu-24.04
21+
timeout-minutes: 5
22+
steps:
23+
# Only trusted default-branch code. Never check out/build the PR in this job.
24+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
25+
with:
26+
ref: ${{ github.event.repository.default_branch }}
27+
persist-credentials: false
28+
path: trusted
29+
- name: Resolve current PR and download bounded data
30+
id: prepare
31+
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
32+
with:
33+
script: |
34+
const publisher = require('./trusted/test/FastExpressionCompiler.Benchmarks/Regression/publish.cjs');
35+
await publisher.prepare({github, context, core, directory: require("node:path").join(process.env.RUNNER_TEMP, "performance-comment")});
36+
- name: Validate and render data without extracting it
37+
if: steps.prepare.outputs.publish == 'true'
38+
shell: bash
39+
run: python3 trusted/test/FastExpressionCompiler.Benchmarks/Regression/publish.py --directory "$RUNNER_TEMP/performance-comment"
40+
- name: Publish current report and outdate older comments
41+
if: steps.prepare.outputs.publish == 'true'
42+
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
43+
with:
44+
script: |
45+
const publisher = require('./trusted/test/FastExpressionCompiler.Benchmarks/Regression/publish.cjs');
46+
await publisher.post({github, context, directory: require("node:path").join(process.env.RUNNER_TEMP, "performance-comment")});

‎.github/workflows/performance.yml‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
name: PR performance
2+
3+
on:
4+
pull_request:
5+
branches: [master]
6+
types: [opened, synchronize, reopened, ready_for_review]
7+
workflow_dispatch:
8+
inputs:
9+
pull_request:
10+
description: Pull request number to compare with its current base
11+
required: true
12+
type: string
13+
runtime:
14+
description: Target runtime
15+
type: choice
16+
options: [net10.0, net8.0, both]
17+
default: net10.0
18+
19+
permissions:
20+
contents: read
21+
pull-requests: read
22+
23+
concurrency:
24+
group: performance-${{ github.event.pull_request.number || inputs.pull_request }}
25+
cancel-in-progress: true
26+
27+
jobs:
28+
revisions:
29+
runs-on: ubuntu-24.04
30+
outputs:
31+
number: ${{ steps.pr.outputs.number }}
32+
base: ${{ steps.pr.outputs.base }}
33+
head: ${{ steps.pr.outputs.head }}
34+
head-repo: ${{ steps.pr.outputs.head-repo }}
35+
runtimes: ${{ steps.pr.outputs.runtimes }}
36+
steps:
37+
- uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
38+
id: pr
39+
env:
40+
REQUESTED_PR: ${{ inputs.pull_request }}
41+
REQUESTED_RUNTIME: ${{ inputs.runtime }}
42+
with:
43+
script: |
44+
const text = process.env.REQUESTED_PR;
45+
if (context.eventName === 'workflow_dispatch' && !/^[1-9][0-9]*$/.test(text)) throw new Error('Invalid PR number');
46+
const pr = context.eventName === 'pull_request' ? context.payload.pull_request :
47+
(await github.rest.pulls.get({...context.repo, pull_number: Number(text)})).data;
48+
if (pr.base.ref !== 'master' || pr.base.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`) throw new Error('Expected a PR targeting master in this repository');
49+
core.setOutput('number', pr.number);
50+
core.setOutput('base', pr.base.sha);
51+
core.setOutput('head', pr.head.sha);
52+
core.setOutput('head-repo', pr.head.repo.full_name);
53+
const runtime = process.env.REQUESTED_RUNTIME || 'net10.0';
54+
if (!['net10.0', 'net8.0', 'both'].includes(runtime)) throw new Error('Invalid runtime');
55+
core.setOutput('runtimes', JSON.stringify(runtime === 'both' ? ['net8.0', 'net10.0'] : [runtime]));
56+
57+
benchmark:
58+
needs: revisions
59+
runs-on: ubuntu-24.04
60+
timeout-minutes: 30
61+
strategy:
62+
fail-fast: false
63+
matrix:
64+
runtime: ${{ fromJSON(needs.revisions.outputs.runtimes) }}
65+
env:
66+
PR_NUMBER: ${{ needs.revisions.outputs.number }}
67+
DOTNET_CLI_TELEMETRY_OPTOUT: 1
68+
DOTNET_NOLOGO: 1
69+
DOTNET_TieredCompilation: 0
70+
DOTNET_ReadyToRun: 0
71+
steps:
72+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
73+
with:
74+
ref: ${{ needs.revisions.outputs.base }}
75+
path: baseline
76+
persist-credentials: false
77+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
78+
with:
79+
repository: ${{ needs.revisions.outputs.head-repo }}
80+
ref: ${{ needs.revisions.outputs.head }}
81+
path: candidate
82+
persist-credentials: false
83+
- uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4
84+
with:
85+
dotnet-version: 10.0.301
86+
# The extended .NET 8 pass needs the runtime as well as the pinned SDK.
87+
- if: matrix.runtime == 'net8.0'
88+
uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4
89+
with:
90+
dotnet-version: 8.0.422
91+
- name: Select one frozen harness
92+
shell: bash
93+
run: |
94+
suite=test/FastExpressionCompiler.Benchmarks/Regression
95+
harness="baseline/$suite"
96+
if [ ! -f "$harness/run.py" ]; then harness="candidate/$suite"; fi
97+
echo "PERFORMANCE_HARNESS=$harness" >> "$GITHUB_ENV"
98+
- name: Harness and policy tests
99+
shell: bash
100+
run: |
101+
python3 -m unittest discover -s "$PERFORMANCE_HARNESS/tests" -v
102+
node --test "$PERFORMANCE_HARNESS/tests/publish.test.cjs"
103+
- name: Benchmark exact base and PR head in both orders
104+
id: measure
105+
continue-on-error: true
106+
env:
107+
RUNTIME: ${{ matrix.runtime }}
108+
shell: bash
109+
run: python3 "$PERFORMANCE_HARNESS/run.py" --baseline baseline --candidate candidate --runtime "$RUNTIME" --output results
110+
- name: Compare (reporting only)
111+
if: always() && steps.measure.outcome != 'skipped'
112+
shell: bash
113+
run: |
114+
status=0
115+
python3 "$PERFORMANCE_HARNESS/compare.py" --input results || status=$?
116+
if [ -f results/report.md ]; then cat results/report.md >> "$GITHUB_STEP_SUMMARY"; fi
117+
exit "$status"
118+
- name: Upload comment data
119+
if: always()
120+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
121+
with:
122+
name: performance-report-${{ matrix.runtime }}-${{ github.run_attempt }}
123+
path: results/report.json
124+
if-no-files-found: warn
125+
retention-days: 14
126+
- name: Upload complete benchmark evidence
127+
if: always()
128+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
129+
with:
130+
name: performance-data-${{ matrix.runtime }}-${{ github.run_attempt }}
131+
path: |
132+
results/context.json
133+
results/sources.json
134+
results/report.md
135+
results/logs
136+
results/runs
137+
if-no-files-found: warn
138+
retention-days: 14

‎test/FastExpressionCompiler.Benchmarks/FastExpressionCompiler.Benchmarks.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
</PropertyGroup>
1818

1919
<ItemGroup>
20+
<Compile Remove="Regression/**/*.cs" />
2021
<PackageReference Include="BenchmarkDotNet" Version="0.15.8"/>
2122
<PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.15.8" />
2223
</ItemGroup>

‎test/FastExpressionCompiler.Benchmarks/Regression/.gitignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
/compiler/
2+
/bin/
3+
/obj/
4+
__pycache__/
5+
/BenchmarkDotNet.Artifacts/

‎test/FastExpressionCompiler.Benchmarks/Regression/Adapter.Light.csproj‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup><TargetFrameworks>net8.0;net10.0</TargetFrameworks><AssemblyName>Regression.Light</AssemblyName><DefineConstants>LIGHT_EXPRESSION</DefineConstants></PropertyGroup>
3+
<ItemGroup><Compile Include="Workloads.cs" /><Reference Include="FastExpressionCompiler.LightExpression"><HintPath>compiler/FastExpressionCompiler.LightExpression.dll</HintPath></Reference></ItemGroup>
4+
</Project>

‎test/FastExpressionCompiler.Benchmarks/Regression/Adapter.Standard.csproj‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup><TargetFrameworks>net8.0;net10.0</TargetFrameworks><AssemblyName>Regression.Standard</AssemblyName></PropertyGroup>
3+
<ItemGroup><Compile Include="Workloads.cs" /><Reference Include="FastExpressionCompiler"><HintPath>compiler/FastExpressionCompiler.dll</HintPath></Reference></ItemGroup>
4+
</Project>

‎test/FastExpressionCompiler.Benchmarks/Regression/Directory.Build.props‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
<Project>
2+
<PropertyGroup>
3+
<LangVersion>latest</LangVersion>
4+
<Nullable>enable</Nullable>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<EnableDefaultCompileItems>false</EnableDefaultCompileItems>
7+
<BaseIntermediateOutputPath>obj/$(MSBuildProjectName)/</BaseIntermediateOutputPath>
8+
<MSBuildProjectExtensionsPath>$(BaseIntermediateOutputPath)</MSBuildProjectExtensionsPath>
9+
<IsTestProject>false</IsTestProject>
10+
</PropertyGroup>
11+
</Project>

‎test/FastExpressionCompiler.Benchmarks/Regression/Program.cs‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
using BenchmarkDotNet.Attributes;
2+
using BenchmarkDotNet.Exporters;
3+
using BenchmarkDotNet.Running;
4+
using BenchmarkDotNet.Configs;
5+
using BenchmarkDotNet.Jobs;
6+
using Perfolizer.Horology;
7+
8+
namespace Regression;
9+
10+
[MemoryDiagnoser]
11+
[JsonExporterAttribute.Full]
12+
public class PerformanceBenchmarks
13+
{
14+
[Params("Standard", "Light")] public string Backend { get; set; } = "Standard";
15+
[Params("Scalar", "Arithmetic", "CallConditional", "Closure")] public string Shape { get; set; } = "Scalar";
16+
private Func<Delegate> compile = null!;
17+
private Func<object> construct = null!;
18+
private Func<int, int>? scalar;
19+
private Func<int, Func<int>>? closure;
20+
private Delegate compiled = null!;
21+
22+
[GlobalSetup]
23+
public void Setup()
24+
{
25+
if (Backend == "Standard")
26+
{
27+
compile = Standard.Workloads.Prepare(Shape);
28+
construct = () => Standard.Workloads.Construct(Shape);
29+
Standard.Workloads.Verify(Shape, compile);
30+
}
31+
else
32+
{
33+
compile = Light.Workloads.Prepare(Shape);
34+
construct = () => Light.Workloads.Construct(Shape);
35+
Light.Workloads.Verify(Shape, compile);
36+
}
37+
compiled = compile();
38+
if (Shape == "Closure") { closure = (Func<int, Func<int>>)compiled; _ = closure(3)(); }
39+
else { scalar = (Func<int, int>)compiled; _ = scalar(3); }
40+
}
41+
42+
[Benchmark] public object Construct() => construct();
43+
[Benchmark] public Delegate Compile() => compile();
44+
[Benchmark]
45+
public int CompileAndFirstInvoke()
46+
{
47+
var fresh = compile();
48+
return Shape == "Closure" ? ((Func<int, Func<int>>)fresh)(3)() : ((Func<int, int>)fresh)(3);
49+
}
50+
51+
// Amortize harness overhead; return a consumed checksum to keep every call observable.
52+
[Benchmark(OperationsPerInvoke = 256)]
53+
public int Invoke()
54+
{
55+
var sum = 0;
56+
if (Shape == "Closure")
57+
{
58+
var f = closure!;
59+
for (var i = 0; i < 256; ++i) sum += f((i & 15) - 8)();
60+
}
61+
else
62+
{
63+
var f = scalar!;
64+
for (var i = 0; i < 256; ++i) sum += f((i & 15) - 8);
65+
}
66+
return sum;
67+
}
68+
}
69+
70+
public static class Program
71+
{
72+
public static int Main(string[] args)
73+
{
74+
if (args.Contains("--verify"))
75+
{
76+
foreach (var backend in new[] { "Standard", "Light" })
77+
foreach (var shape in new[] { "Scalar", "Arithmetic", "CallConditional", "Closure" })
78+
{
79+
var benchmark = new PerformanceBenchmarks { Backend = backend, Shape = shape };
80+
benchmark.Setup();
81+
var first = shape switch { "Scalar" or "Closure" => 4, "Arithmetic" => 96, _ => 7 };
82+
var batched = shape switch { "Scalar" or "Closure" => 128, "Arithmetic" => -4096, _ => 1216 };
83+
if (benchmark.CompileAndFirstInvoke() != first || benchmark.Invoke() != batched || benchmark.Construct() == null)
84+
throw new InvalidOperationException("Incorrect benchmark driver: " + backend + "/" + shape);
85+
}
86+
Console.WriteLine("All 8 workload/backend behavior checks passed.");
87+
return 0;
88+
}
89+
var smoke = args.Contains("--smoke");
90+
var job = smoke ? Job.Dry : Job.Default.WithLaunchCount(1).WithWarmupCount(2).WithIterationCount(5).WithIterationTime(TimeInterval.FromMilliseconds(100));
91+
var config = ManualConfig.Create(DefaultConfig.Instance).AddJob(job.WithId(smoke ? "Smoke" : "Regression"));
92+
var summaries = BenchmarkSwitcher.FromTypes(new[] { typeof(PerformanceBenchmarks) }).Run(args.Where(a => a != "--smoke").ToArray(), config).ToArray();
93+
return summaries.Length == 0 || summaries.Any(s => s.HasCriticalValidationErrors || s.Reports.Any(r => !r.Success)) ? 1 : 0;
94+
}
95+
}

0 commit comments

Comments
 (0)