From 7f83a2b3369e43922e640130ccfcc16a317ef39a Mon Sep 17 00:00:00 2001 From: Joan Leon Date: Mon, 26 Oct 2026 00:56:33 +0100 Subject: [PATCH] feat(cli): list every budget and its verdict in the reports The reports list every budget the run was given, with its value and verdict: passed, exceeded with by how much, or not measured with why. The human and markdown reports have a Budgets section, the JSON output and the body of --report-to a budgets field, also in each combination of a matrix, and a matrix lists the failed budgets with their combination under its table. A pull request comment says which budget failed, which only stderr said before. cli/src/reporters/budgets-view.js shapes the rows for both reports, and bin.js keeps the verdicts it already computes for the exit code. --- cli/CHANGELOG.md | 1 + cli/README.md | 2 +- cli/src/bin.js | 11 +++++-- cli/src/reporters/budgets-view.js | 38 ++++++++++++++++++++++++ cli/src/reporters/human.js | 21 ++++++++++++- cli/src/reporters/markdown.js | 15 +++++++++- cli/tests/e2e/budgets.test.js | 13 ++++++++ cli/tests/e2e/matrix.test.js | 4 ++- cli/tests/e2e/report-to.test.js | 8 +++++ cli/tests/unit/reporters.test.js | 49 +++++++++++++++++++++++++++++++ content/CLI.mdx | 2 +- 11 files changed, 157 insertions(+), 7 deletions(-) create mode 100644 cli/src/reporters/budgets-view.js diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 0ae066e5..c7ac4005 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -9,6 +9,7 @@ Format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). The CL ## [Unreleased] ### Added +- The reports list every budget with its value and verdict (`passed`, `exceeded` with by how much, or `not measured` with why): a Budgets section in the human and markdown reports, and `budgets` in the JSON output, in each combination of a matrix and in the body of `--report-to`. A matrix lists the failed budgets with their combination under its table. A pull request comment says which budget failed, which only stderr said before. - `--budget-fcp`, `--budget-ttfb` and `--budget-inp`, next to `--budget-lcp` and `--budget-cls`: a budget for each metric the CLI measures. The interaction workflow measures INP, in a step `INP` that needs a click or a key press like `interactions` and `input-latency`. The GitHub Action takes `budget-fcp`, `budget-ttfb` and `budget-inp`. ### Changed diff --git a/cli/README.md b/cli/README.md index d8ea8652..00c5be42 100644 --- a/cli/README.md +++ b/cli/README.md @@ -300,7 +300,7 @@ CI gating: npx webperf-snippets https://example.com --budget-lcp 2500 --budget-cls 0.1 ``` -Each budget compares the value measured, or the median with `--runs`, and fails when the value is above it. `--budget-lcp` and `--budget-cls` need the `core-web-vitals` workflow, `--budget-fcp` the `loading` workflow, `--budget-ttfb` the `loading` or `audit` workflow, and `--budget-inp` the `interaction` workflow with a click or a key press; `--snippet` with the snippet of the metric works too. A budget of a metric that the run does not measure exits with code `2` before the page loads, and one whose metric was not measured, such as INP without interactions, exits with code `1`: a budget never passes unchecked. Each failed budget is written to stderr, also with `--json`. +Each budget compares the value measured, or the median with `--runs`, and fails when the value is above it. `--budget-lcp` and `--budget-cls` need the `core-web-vitals` workflow, `--budget-fcp` the `loading` workflow, `--budget-ttfb` the `loading` or `audit` workflow, and `--budget-inp` the `interaction` workflow with a click or a key press; `--snippet` with the snippet of the metric works too. A budget of a metric that the run does not measure exits with code `2` before the page loads, and one whose metric was not measured, such as INP without interactions, exits with code `1`: a budget never passes unchecked. The report lists every budget with its value and verdict, `passed`, `exceeded` with by how much, or `not measured` with why: a Budgets section in the human and markdown reports, and `budgets` in the JSON output and in the body of `--report-to`. A matrix lists the failed budgets with their combination under its table. Each failed budget is also written to stderr, with `--json` too. ### Options diff --git a/cli/src/bin.js b/cli/src/bin.js index 6e8b890b..69b9c804 100755 --- a/cli/src/bin.js +++ b/cli/src/bin.js @@ -318,6 +318,9 @@ async function main() { for (const warning of payload.baseline.warnings) process.stderr.write(`Warning: ${warning}\n`); } + // The verdict of each budget goes in the report, so a pull request comment says which one failed + if (Object.keys(budgets).length > 0) payload.budgets = checkBudgets(payload, budgets); + let output; if (values.markdown) { output = reportMarkdown(payload); @@ -339,6 +342,7 @@ async function main() { timestamp: new Date().toISOString(), navMs: payload.navMs, results: payload.results, + ...(payload.budgets ? { budgets: payload.budgets } : {}), meta: { viewport: viewportName, waitMs, ...(payload.throttling ? { throttling: payload.throttling } : {}), ...(payload.runs ? { runs: payload.runs } : {}), cli_version: cliVersion }, }, }); @@ -350,7 +354,7 @@ async function main() { } // Exit codes. stderr names the budgets that failed, also with --json, whose report is on stdout - const violations = violationsOf(checkBudgets(payload, budgets)); + const violations = violationsOf(payload.budgets ?? []); if (violations.length > 0) { for (const v of violations) process.stderr.write(`Budget violation: ${v}\n`); process.exit(1); @@ -424,6 +428,8 @@ async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, repo for (const warning of matrixComparisonView(root, combinations).warnings) process.stderr.write(`Warning: ${warning}\n`); } + if (Object.keys(budgets).length > 0) for (const c of combinations.filter((c) => !c.error)) c.budgets = checkBudgets(c, budgets); + const withBaseline = comparison ? { baseline: comparison } : {}; const report = { matrix: matrixPath, runs: matrix.runs, combinations, verbose: values.verbose, ...withBaseline }; const output = values.markdown ? reportMatrixMarkdown(report) : values.json ? reportJson({ matrix: matrixPath, runs: matrix.runs, ...withBaseline, combinations }) : reportMatrixHuman(report); @@ -443,6 +449,7 @@ async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, repo timestamp: new Date().toISOString(), navMs: c.navMs, results: c.results, + ...(c.budgets ? { budgets: c.budgets } : {}), meta: { viewport: viewportOf.get(c.id), device: c.device, @@ -462,7 +469,7 @@ async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, repo } } - const violations = combinations.filter((c) => !c.error).flatMap((c) => violationsOf(checkBudgets(c, budgets)).map((v) => `${v} (${c.id})`)); + const violations = combinations.flatMap((c) => violationsOf(c.budgets ?? []).map((v) => `${v} (${c.id})`)); for (const v of violations) process.stderr.write(`Budget violation: ${v}\n`); const failed = combinations.some((c) => c.error || blockingErrors(c.results).length > 0); // Without --fail-on-regression the comparison only informs diff --git a/cli/src/reporters/budgets-view.js b/cli/src/reporters/budgets-view.js new file mode 100644 index 00000000..2e1b5dc4 --- /dev/null +++ b/cli/src/reporters/budgets-view.js @@ -0,0 +1,38 @@ +// The budgets as the reports show them, each report with its own format of a value: a row per budget, +// passed or not, with by how much it was exceeded or why its metric was not measured. + +const difference = (budget) => { + const over = budget.value - budget.budget; + return budget.unit === "ms" ? Math.round(over) : Math.round(over * 10000) / 10000; +}; + +export function budgetRows(budgets, format) { + return budgets.map((b) => ({ + metric: b.metric, + value: b.value == null ? "·" : format(b.value, b.unit), + budget: format(b.budget, b.unit), + passed: b.verdict === "passed", + verdict: + b.verdict === "exceeded" + ? `exceeded by ${format(difference(b), b.unit)}` + : b.verdict === "not measured" + ? `not measured${b.note ? `: ${b.note}` : ""}` + : "passed", + })); +} + +// The failed budgets of a matrix, each named with its combination, or null when no combination had +// budgets to check +export function matrixBudgetFailures(combinations, format) { + const checked = combinations.filter((c) => Array.isArray(c.budgets) && c.budgets.length > 0); + if (checked.length === 0) return null; + return checked.flatMap((c) => + c.budgets + .filter((b) => b.verdict !== "passed") + .map((b) => + b.verdict === "exceeded" + ? `${c.id}: ${b.metric} ${format(b.value, b.unit)} exceeds budget ${format(b.budget, b.unit)}` + : `${c.id}: ${b.metric} was not measured${b.note ? ` (${b.note})` : ""}` + ) + ); +} diff --git a/cli/src/reporters/human.js b/cli/src/reporters/human.js index 80ccc0d6..9d679eab 100644 --- a/cli/src/reporters/human.js +++ b/cli/src/reporters/human.js @@ -4,6 +4,7 @@ import { describeThrottling } from "../throttling.js"; import { describeRuns } from "../runs.js"; import { matrixView } from "./matrix-view.js"; import { baselineView } from "./baseline-view.js"; +import { budgetRows, matrixBudgetFailures } from "./budgets-view.js"; const RATING_ICON = { good: "🟢", @@ -223,6 +224,16 @@ function renderRuns(runs) { const VERDICT_STYLE = { regressed: "red", improved: "green", unchanged: "dim", "not measured": "dim", new: "cyan" }; +// Every budget the run was given, passed or not, so the report says which one decided the exit code +function renderBudgets(budgets) { + const lines = ["", styleText("bold", "Budgets")]; + for (const row of budgetRows(budgets, formatValue)) { + const mark = row.passed ? styleText("green", "✓") : styleText("red", "✗"); + lines.push(` ${mark} ${pad(row.metric, 6)}${pad(row.value, 10)}${pad(`budget ${row.budget}`, 18)}${styleText(row.passed ? "green" : "red", row.verdict)}`); + } + return lines; +} + // The comparison with --baseline, after the report: its warnings first, since they say how far to // trust the table, then a row per metric and the errors that appeared or went away function renderBaseline(comparison) { @@ -240,7 +251,7 @@ function renderBaseline(comparison) { return lines; } -export function reportHuman({ url, navMs, results, pageErrors, verbose, crux, throttling, runs, baseline, title = `WebPerf Snippets — ${url}` }) { +export function reportHuman({ url, navMs, results, pageErrors, verbose, crux, throttling, runs, baseline, budgets, title = `WebPerf Snippets — ${url}` }) { const lines = []; lines.push(styleText(["bold"], title)); lines.push(styleText("dim", `Navigated in ${navMs}ms`)); @@ -257,6 +268,7 @@ export function reportHuman({ url, navMs, results, pageErrors, verbose, crux, th lines.push(styleText("yellow", "Page errors during run:")); for (const err of pageErrors) lines.push(` ${styleText("dim", err)}`); } + if (budgets?.length) lines.push(...renderBudgets(budgets)); if (baseline) lines.push(...renderBaseline(baseline)); return lines.join("\n"); } @@ -294,6 +306,13 @@ export function reportMatrixHuman(report) { for (const e of errors) lines.push(` ${styleText("red", "✗")} ${e}`); } + const budgetFailures = matrixBudgetFailures(report.combinations, formatValue); + if (budgetFailures) { + lines.push("", styleText("bold", "Budgets")); + for (const f of budgetFailures) lines.push(` ${styleText("red", "✗")} ${f}`); + if (!budgetFailures.length) lines.push(` ${styleText("green", "✓ Every budget passed")}`); + } + if (comparison) { lines.push("", styleText("bold", `Compared with ${comparison.file}`)); for (const warning of comparison.warnings) lines.push(` ${styleText("yellow", `Warning: ${warning}`)}`); diff --git a/cli/src/reporters/markdown.js b/cli/src/reporters/markdown.js index efea1506..1c74152a 100644 --- a/cli/src/reporters/markdown.js +++ b/cli/src/reporters/markdown.js @@ -3,6 +3,7 @@ import { describeThrottling } from "../throttling.js"; import { describeRuns } from "../runs.js"; import { matrixView } from "./matrix-view.js"; import { baselineView } from "./baseline-view.js"; +import { budgetRows, matrixBudgetFailures } from "./budgets-view.js"; const RATING_ICON = { good: "✅", @@ -110,7 +111,13 @@ function renderBaseline(comparison) { } // With a null title, the report has no heading, as inside the details block of a matrix -export function reportMarkdown({ url, navMs, results, pageErrors, crux, throttling, runs, baseline, title = `WebPerf Results — ${url}` }) { +// Every budget the run was given, passed or not: the comment of a pull request says which one failed +function renderBudgets(budgets) { + const rows = budgetRows(budgets, formatValue).map((row) => `| ${row.metric} | ${row.value} | ${row.budget} | ${row.passed ? "✅" : "❌"} ${row.verdict} |`); + return ["", "### Budgets", "| Metric | Value | Budget | Verdict |", "|--------|-------|--------|---------|", ...rows]; +} + +export function reportMarkdown({ url, navMs, results, pageErrors, crux, throttling, runs, baseline, budgets, title = `WebPerf Results — ${url}` }) { const lines = []; if (title) lines.push(`## ${title}`); lines.push(`> Navigated in ${navMs}ms`); @@ -146,6 +153,7 @@ export function reportMarkdown({ url, navMs, results, pageErrors, crux, throttli for (const err of pageErrors) lines.push(`- ${err}`); } + if (budgets?.length) lines.push(...renderBudgets(budgets)); if (baseline) lines.push(...renderBaseline(baseline)); return lines.join("\n"); @@ -178,6 +186,11 @@ export function reportMatrixMarkdown(report) { lines.push("", "### Errors", ...errors.map((e) => `- ${e}`)); } + const budgetFailures = matrixBudgetFailures(report.combinations, formatValue); + if (budgetFailures) { + lines.push("", "### Budgets", ...(budgetFailures.length ? budgetFailures.map((f) => `- ❌ ${f}`) : ["✅ Every budget passed"])); + } + if (comparison) { lines.push("", `### Compared with \`${comparison.file}\``); if (comparison.warnings.length) lines.push(...comparison.warnings.flatMap((w, i) => [...(i ? [">"] : []), `> ⚠️ ${w}`]), ""); diff --git a/cli/tests/e2e/budgets.test.js b/cli/tests/e2e/budgets.test.js index d50fb698..b0e063ec 100644 --- a/cli/tests/e2e/budgets.test.js +++ b/cli/tests/e2e/budgets.test.js @@ -98,3 +98,16 @@ describe("--budget-fcp and --budget-ttfb", () => { expect(tight.stderr).toMatch(/Budget violation: TTFB [\d.]+ms exceeds budget 0ms/); }, 120000); }); + +describe("the budgets in the reports", () => { + it("gives every budget its verdict in the JSON output and in the markdown report", async () => { + const json = await cli(base, "--wait", WAIT_MS, "--budget-lcp", "60000", "--budget-cls", "0", "--json"); + const { budgets } = JSON.parse(json.stdout); + expect(budgets, json.said).toEqual([ + expect.objectContaining({ metric: "LCP", budget: 60000, unit: "ms", verdict: "passed" }), + expect.objectContaining({ metric: "CLS", budget: 0, unit: "score" }), + ]); + const markdown = await cli(base, "--wait", WAIT_MS, "--budget-lcp", "60000", "--markdown"); + expect(markdown.stdout, markdown.said).toMatch(/### Budgets\n\| Metric \| Value \| Budget \| Verdict \|[\s\S]*\| LCP \| [\d.]+m?s \| 60\.00s \| ✅ passed \|/); + }, 120000); +}); diff --git a/cli/tests/e2e/matrix.test.js b/cli/tests/e2e/matrix.test.js index 46587821..f986fe9c 100644 --- a/cli/tests/e2e/matrix.test.js +++ b/cli/tests/e2e/matrix.test.js @@ -152,10 +152,12 @@ describe("--matrix", () => { }, 120000); it("compares the budgets with the median of each combination, and names the combination", async () => { - const { code, stderr, said: output } = await cli("--matrix", writeMatrix(clientCase()), "--budget-lcp", "250"); + const { code, stdout, stderr, said: output } = await cli("--matrix", writeMatrix(clientCase()), "--budget-lcp", "250"); expect(code, output).toBe(1); // The phone loads the document with 300 ms of latency, so its LCP is over the budget expect(stderr).toMatch(/Budget violation: LCP \d+ms exceeds budget 250ms \(home · phone · anon\)/); + // The report says it too, under the table + expect(stdout).toMatch(/Budgets\n[\s\S]*home · phone · anon: LCP \d+ms exceeds budget 250ms/); }, 120000); it("warns about a header of the file that looks like a secret", async () => { diff --git a/cli/tests/e2e/report-to.test.js b/cli/tests/e2e/report-to.test.js index 8b32e3b6..dc679ad4 100644 --- a/cli/tests/e2e/report-to.test.js +++ b/cli/tests/e2e/report-to.test.js @@ -96,6 +96,14 @@ describe("--report-to", () => { expect(body.meta).toEqual({ viewport: "desktop", waitMs: 600, cli_version: CLI_VERSION }); }, 30000); + it("sends the verdict of each budget, and no budgets field without them", async () => { + await cli(["--report-to", reportUrl, "--budget-lcp", "60000"]); + expect(received[0].body.budgets).toEqual([expect.objectContaining({ metric: "LCP", budget: 60000, verdict: "passed", unit: "ms" })]); + received.length = 0; + await cli(["--report-to", reportUrl]); + expect(received[0].body).not.toHaveProperty("budgets"); + }, 60000); + it("sends --api-key as a Bearer token", async () => { await cli(["--report-to", reportUrl, "--api-key", "key-from-flag"]); expect(received[0].headers.authorization).toBe("Bearer key-from-flag"); diff --git a/cli/tests/unit/reporters.test.js b/cli/tests/unit/reporters.test.js index a2457244..8dd24096 100644 --- a/cli/tests/unit/reporters.test.js +++ b/cli/tests/unit/reporters.test.js @@ -612,3 +612,52 @@ describe("the human report for the Interaction snippets", () => { expect(output).not.toMatch(/·\s*$/m); }); }); + +describe("the budgets in the reports", () => { + const budgets = [ + { metric: "LCP", budget: 2500, value: 1980, unit: "ms", verdict: "passed" }, + { metric: "INP", budget: 200, value: 260, unit: "ms", verdict: "exceeded" }, + { metric: "CLS", budget: 0.1, value: 0.12, unit: "score", verdict: "exceeded" }, + { metric: "TTFB", budget: 800, value: null, unit: "ms", verdict: "not measured", note: "No navigation entry" }, + ]; + const withBudgets = { ...samplePayload, budgets }; + + it("lists every budget in the human report, passed or not, with by how much it was exceeded or why it was not measured", () => { + const output = reportHuman(withBudgets); + expect(output).toContain("Budgets"); + expect(output).toMatch(/LCP\s+1\.98s\s+budget 2\.50s\s+passed/); + expect(output).toMatch(/INP\s+260ms\s+budget 200ms\s+exceeded by 60ms/); + expect(output).toMatch(/CLS\s+0\.1200\s+budget 0\.1000\s+exceeded by 0\.0200/); + expect(output).toMatch(/TTFB\s+·\s+budget 800ms\s+not measured: No navigation entry/); + }); + + it("gives the markdown report a table of the budgets, for the comment of a pull request", () => { + const output = reportMarkdown(withBudgets); + expect(output).toContain("### Budgets"); + expect(output).toContain("| Metric | Value | Budget | Verdict |"); + expect(output).toContain("| LCP | 1.98s | 2.50s | ✅ passed |"); + expect(output).toContain("| INP | 260ms | 200ms | ❌ exceeded by 60ms |"); + expect(output).toContain("| TTFB | · | 800ms | ❌ not measured: No navigation entry |"); + }); + + it("adds no section without budgets", () => { + expect(reportHuman(samplePayload)).not.toContain("Budgets"); + expect(reportMarkdown(samplePayload)).not.toContain("Budgets"); + }); + + it("keeps the budgets in the JSON report", () => { + expect(JSON.parse(reportJson(withBudgets)).budgets).toEqual(budgets); + }); + + it("names the combination of each failed budget in a matrix, or says that every budget passed", () => { + const lcp = (value, verdict) => [{ metric: "LCP", budget: 2500, value, unit: "ms", verdict }]; + const [first, second, failed] = matrix.combinations; + const failing = { ...matrix, combinations: [{ ...first, budgets: lcp(1980, "passed") }, { ...second, budgets: lcp(2710, "exceeded") }, failed] }; + expect(reportMatrixHuman(failing)).toMatch(/Budgets[\s\S]*home · mobile · consent: LCP 2\.71s exceeds budget 2\.50s/); + expect(reportMatrixMarkdown(failing)).toMatch(/### Budgets\n- ❌ home · mobile · consent: LCP 2\.71s exceeds budget 2\.50s/); + const passing = { ...matrix, combinations: [{ ...first, budgets: lcp(1980, "passed") }, failed] }; + expect(reportMatrixHuman(passing)).toMatch(/Budgets[\s\S]*Every budget passed/); + expect(reportMatrixMarkdown(passing)).toContain("✅ Every budget passed"); + expect(reportMatrixMarkdown(matrix)).not.toContain("Budgets"); + }); +}); diff --git a/content/CLI.mdx b/content/CLI.mdx index 144536f9..8c3afd38 100644 --- a/content/CLI.mdx +++ b/content/CLI.mdx @@ -196,7 +196,7 @@ Gate CI on performance budgets: npx webperf-snippets https://example.com --budget-lcp 2500 --budget-cls 0.1 ``` -Each budget compares the value measured, or the median with `--runs`, and fails when the value is above it. `--budget-lcp` and `--budget-cls` need the `core-web-vitals` workflow, `--budget-fcp` the `loading` workflow, `--budget-ttfb` the `loading` or `audit` workflow, and `--budget-inp` the `interaction` workflow with a click or a key press; `--snippet` with the snippet of the metric works too. A budget of a metric that the run does not measure exits with code `2` before the page loads, and one whose metric was not measured, such as INP without interactions, exits with code `1`: a budget never passes unchecked. Each failed budget is written to stderr, also with `--json`. +Each budget compares the value measured, or the median with `--runs`, and fails when the value is above it. `--budget-lcp` and `--budget-cls` need the `core-web-vitals` workflow, `--budget-fcp` the `loading` workflow, `--budget-ttfb` the `loading` or `audit` workflow, and `--budget-inp` the `interaction` workflow with a click or a key press; `--snippet` with the snippet of the metric works too. A budget of a metric that the run does not measure exits with code `2` before the page loads, and one whose metric was not measured, such as INP without interactions, exits with code `1`: a budget never passes unchecked. The report lists every budget with its value and verdict, `passed`, `exceeded` with by how much, or `not measured` with why: a Budgets section in the human and markdown reports, and `budgets` in the JSON output and in the body of `--report-to`. A matrix lists the failed budgets with their combination under its table. Each failed budget is also written to stderr, with `--json` too. Run the structural audit workflow: