feat(cli): list every budget and its verdict in the reports

cli/CHANGELOG.md

@@ -9,6 +9,7 @@ Format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). The CL
 ## [Unreleased]
 
 ### Added
+- The reports list every budget with its value and verdict (`passed`, `exceeded` with by how much, or `not measured` with why): a Budgets section in the human and markdown reports, and `budgets` in the JSON output, in each combination of a matrix and in the body of `--report-to`. A matrix lists the failed budgets with their combination under its table. A pull request comment says which budget failed, which only stderr said before.
 - `--budget-fcp`, `--budget-ttfb` and `--budget-inp`, next to `--budget-lcp` and `--budget-cls`: a budget for each metric the CLI measures. The interaction workflow measures INP, in a step `INP` that needs a click or a key press like `interactions` and `input-latency`. The GitHub Action takes `budget-fcp`, `budget-ttfb` and `budget-inp`.
 
 ### Changed

cli/README.md

@@ -300,7 +300,7 @@ CI gating:
 npx webperf-snippets https://example.com --budget-lcp 2500 --budget-cls 0.1
 ```
 
-Each budget compares the value measured, or the median with `--runs`, and fails when the value is above it. `--budget-lcp` and `--budget-cls` need the `core-web-vitals` workflow, `--budget-fcp` the `loading` workflow, `--budget-ttfb` the `loading` or `audit` workflow, and `--budget-inp` the `interaction` workflow with a click or a key press; `--snippet` with the snippet of the metric works too. A budget of a metric that the run does not measure exits with code `2` before the page loads, and one whose metric was not measured, such as INP without interactions, exits with code `1`: a budget never passes unchecked. Each failed budget is written to stderr, also with `--json`.
+Each budget compares the value measured, or the median with `--runs`, and fails when the value is above it. `--budget-lcp` and `--budget-cls` need the `core-web-vitals` workflow, `--budget-fcp` the `loading` workflow, `--budget-ttfb` the `loading` or `audit` workflow, and `--budget-inp` the `interaction` workflow with a click or a key press; `--snippet` with the snippet of the metric works too. A budget of a metric that the run does not measure exits with code `2` before the page loads, and one whose metric was not measured, such as INP without interactions, exits with code `1`: a budget never passes unchecked. The report lists every budget with its value and verdict, `passed`, `exceeded` with by how much, or `not measured` with why: a Budgets section in the human and markdown reports, and `budgets` in the JSON output and in the body of `--report-to`. A matrix lists the failed budgets with their combination under its table. Each failed budget is also written to stderr, with `--json` too.
 
 ### Options
 

cli/src/bin.js

@@ -318,6 +318,9 @@ async function main() {
     for (const warning of payload.baseline.warnings) process.stderr.write(`Warning: ${warning}\n`);
   }
 
+  // The verdict of each budget goes in the report, so a pull request comment says which one failed
+  if (Object.keys(budgets).length > 0) payload.budgets = checkBudgets(payload, budgets);
+
   let output;
   if (values.markdown) {
     output = reportMarkdown(payload);
@@ -339,6 +342,7 @@ async function main() {
         timestamp: new Date().toISOString(),
         navMs: payload.navMs,
         results: payload.results,
+        ...(payload.budgets ? { budgets: payload.budgets } : {}),
         meta: { viewport: viewportName, waitMs, ...(payload.throttling ? { throttling: payload.throttling } : {}), ...(payload.runs ? { runs: payload.runs } : {}), cli_version: cliVersion },
       },
     });
@@ -350,7 +354,7 @@ async function main() {
   }
 
   // Exit codes. stderr names the budgets that failed, also with --json, whose report is on stdout
-  const violations = violationsOf(checkBudgets(payload, budgets));
+  const violations = violationsOf(payload.budgets ?? []);
   if (violations.length > 0) {
     for (const v of violations) process.stderr.write(`Budget violation: ${v}\n`);
     process.exit(1);
@@ -424,6 +428,8 @@ async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, repo
     for (const warning of matrixComparisonView(root, combinations).warnings) process.stderr.write(`Warning: ${warning}\n`);
   }
 
+  if (Object.keys(budgets).length > 0) for (const c of combinations.filter((c) => !c.error)) c.budgets = checkBudgets(c, budgets);
+
   const withBaseline = comparison ? { baseline: comparison } : {};
   const report = { matrix: matrixPath, runs: matrix.runs, combinations, verbose: values.verbose, ...withBaseline };
   const output = values.markdown ? reportMatrixMarkdown(report) : values.json ? reportJson({ matrix: matrixPath, runs: matrix.runs, ...withBaseline, combinations }) : reportMatrixHuman(report);
@@ -443,6 +449,7 @@ async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, repo
           timestamp: new Date().toISOString(),
           navMs: c.navMs,
           results: c.results,
+          ...(c.budgets ? { budgets: c.budgets } : {}),
           meta: {
             viewport: viewportOf.get(c.id),
             device: c.device,
@@ -462,7 +469,7 @@ async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, repo
     }
   }
 
-  const violations = combinations.filter((c) => !c.error).flatMap((c) => violationsOf(checkBudgets(c, budgets)).map((v) => `${v} (${c.id})`));
+  const violations = combinations.flatMap((c) => violationsOf(c.budgets ?? []).map((v) => `${v} (${c.id})`));
   for (const v of violations) process.stderr.write(`Budget violation: ${v}\n`);
   const failed = combinations.some((c) => c.error || blockingErrors(c.results).length > 0);
   // Without --fail-on-regression the comparison only informs

cli/src/reporters/budgets-view.js

@@ -0,0 +1,38 @@
+// The budgets as the reports show them, each report with its own format of a value: a row per budget,
+// passed or not, with by how much it was exceeded or why its metric was not measured.
+
+const difference = (budget) => {
+  const over = budget.value - budget.budget;
+  return budget.unit === "ms" ? Math.round(over) : Math.round(over * 10000) / 10000;
+};
+
+export function budgetRows(budgets, format) {
+  return budgets.map((b) => ({
+    metric: b.metric,
+    value: b.value == null ? "·" : format(b.value, b.unit),
+    budget: format(b.budget, b.unit),
+    passed: b.verdict === "passed",
+    verdict:
+      b.verdict === "exceeded"
+        ? `exceeded by ${format(difference(b), b.unit)}`
+        : b.verdict === "not measured"
+          ? `not measured${b.note ? `: ${b.note}` : ""}`
+          : "passed",
+  }));
+}
+
+// The failed budgets of a matrix, each named with its combination, or null when no combination had
+// budgets to check
+export function matrixBudgetFailures(combinations, format) {
+  const checked = combinations.filter((c) => Array.isArray(c.budgets) && c.budgets.length > 0);
+  if (checked.length === 0) return null;
+  return checked.flatMap((c) =>
+    c.budgets
+      .filter((b) => b.verdict !== "passed")
+      .map((b) =>
+        b.verdict === "exceeded"
+          ? `${c.id}: ${b.metric} ${format(b.value, b.unit)} exceeds budget ${format(b.budget, b.unit)}`
+          : `${c.id}: ${b.metric} was not measured${b.note ? ` (${b.note})` : ""}`
+      )
+  );
+}

cli/src/reporters/human.js

@@ -4,6 +4,7 @@ import { describeThrottling } from "../throttling.js";
 import { describeRuns } from "../runs.js";
 import { matrixView } from "./matrix-view.js";
 import { baselineView } from "./baseline-view.js";
+import { budgetRows, matrixBudgetFailures } from "./budgets-view.js";
 
 const RATING_ICON = {
   good: "🟢",
@@ -223,6 +224,16 @@ function renderRuns(runs) {
 
 const VERDICT_STYLE = { regressed: "red", improved: "green", unchanged: "dim", "not measured": "dim", new: "cyan" };
 
+// Every budget the run was given, passed or not, so the report says which one decided the exit code
+function renderBudgets(budgets) {
+  const lines = ["", styleText("bold", "Budgets")];
+  for (const row of budgetRows(budgets, formatValue)) {
+    const mark = row.passed ? styleText("green", "✓") : styleText("red", "✗");
+    lines.push(`  ${mark} ${pad(row.metric, 6)}${pad(row.value, 10)}${pad(`budget ${row.budget}`, 18)}${styleText(row.passed ? "green" : "red", row.verdict)}`);
+  }
+  return lines;
+}
+
 // The comparison with --baseline, after the report: its warnings first, since they say how far to
 // trust the table, then a row per metric and the errors that appeared or went away
 function renderBaseline(comparison) {
@@ -240,7 +251,7 @@ function renderBaseline(comparison) {
   return lines;
 }
 
-export function reportHuman({ url, navMs, results, pageErrors, verbose, crux, throttling, runs, baseline, title = `WebPerf Snippets — ${url}` }) {
+export function reportHuman({ url, navMs, results, pageErrors, verbose, crux, throttling, runs, baseline, budgets, title = `WebPerf Snippets — ${url}` }) {
   const lines = [];
   lines.push(styleText(["bold"], title));
   lines.push(styleText("dim", `Navigated in ${navMs}ms`));
@@ -257,6 +268,7 @@ export function reportHuman({ url, navMs, results, pageErrors, verbose, crux, th
     lines.push(styleText("yellow", "Page errors during run:"));
     for (const err of pageErrors) lines.push(`  ${styleText("dim", err)}`);
   }
+  if (budgets?.length) lines.push(...renderBudgets(budgets));
   if (baseline) lines.push(...renderBaseline(baseline));
   return lines.join("\n");
 }
@@ -294,6 +306,13 @@ export function reportMatrixHuman(report) {
     for (const e of errors) lines.push(`  ${styleText("red", "✗")} ${e}`);
   }
 
+  const budgetFailures = matrixBudgetFailures(report.combinations, formatValue);
+  if (budgetFailures) {
+    lines.push("", styleText("bold", "Budgets"));
+    for (const f of budgetFailures) lines.push(`  ${styleText("red", "✗")} ${f}`);
+    if (!budgetFailures.length) lines.push(`  ${styleText("green", "✓ Every budget passed")}`);
+  }
+
   if (comparison) {
     lines.push("", styleText("bold", `Compared with ${comparison.file}`));
     for (const warning of comparison.warnings) lines.push(`  ${styleText("yellow", `Warning: ${warning}`)}`);

cli/src/reporters/markdown.js

@@ -3,6 +3,7 @@ import { describeThrottling } from "../throttling.js";
 import { describeRuns } from "../runs.js";
 import { matrixView } from "./matrix-view.js";
 import { baselineView } from "./baseline-view.js";
+import { budgetRows, matrixBudgetFailures } from "./budgets-view.js";
 
 const RATING_ICON = {
   good: "✅",
@@ -110,7 +111,13 @@ function renderBaseline(comparison) {
 }
 
 // With a null title, the report has no heading, as inside the details block of a matrix
-export function reportMarkdown({ url, navMs, results, pageErrors, crux, throttling, runs, baseline, title = `WebPerf Results — ${url}` }) {
+// Every budget the run was given, passed or not: the comment of a pull request says which one failed
+function renderBudgets(budgets) {
+  const rows = budgetRows(budgets, formatValue).map((row) => `| ${row.metric} | ${row.value} | ${row.budget} | ${row.passed ? "✅" : "❌"} ${row.verdict} |`);
+  return ["", "### Budgets", "| Metric | Value | Budget | Verdict |", "|--------|-------|--------|---------|", ...rows];
+}
+
+export function reportMarkdown({ url, navMs, results, pageErrors, crux, throttling, runs, baseline, budgets, title = `WebPerf Results — ${url}` }) {
   const lines = [];
   if (title) lines.push(`## ${title}`);
   lines.push(`> Navigated in ${navMs}ms`);
@@ -146,6 +153,7 @@ export function reportMarkdown({ url, navMs, results, pageErrors, crux, throttli
     for (const err of pageErrors) lines.push(`- ${err}`);
   }
 
+  if (budgets?.length) lines.push(...renderBudgets(budgets));
   if (baseline) lines.push(...renderBaseline(baseline));
 
   return lines.join("\n");
@@ -178,6 +186,11 @@ export function reportMatrixMarkdown(report) {
     lines.push("", "### Errors", ...errors.map((e) => `- ${e}`));
   }
 
+  const budgetFailures = matrixBudgetFailures(report.combinations, formatValue);
+  if (budgetFailures) {
+    lines.push("", "### Budgets", ...(budgetFailures.length ? budgetFailures.map((f) => `- ❌ ${f}`) : ["✅ Every budget passed"]));
+  }
+
   if (comparison) {
     lines.push("", `### Compared with \`${comparison.file}\``);
     if (comparison.warnings.length) lines.push(...comparison.warnings.flatMap((w, i) => [...(i ? [">"] : []), `> ⚠️ ${w}`]), "");

cli/tests/e2e/budgets.test.js

@@ -98,3 +98,16 @@ describe("--budget-fcp and --budget-ttfb", () => {
     expect(tight.stderr).toMatch(/Budget violation: TTFB [\d.]+ms exceeds budget 0ms/);
   }, 120000);
 });
+
+describe("the budgets in the reports", () => {
+  it("gives every budget its verdict in the JSON output and in the markdown report", async () => {
+    const json = await cli(base, "--wait", WAIT_MS, "--budget-lcp", "60000", "--budget-cls", "0", "--json");
+    const { budgets } = JSON.parse(json.stdout);
+    expect(budgets, json.said).toEqual([
+      expect.objectContaining({ metric: "LCP", budget: 60000, unit: "ms", verdict: "passed" }),
+      expect.objectContaining({ metric: "CLS", budget: 0, unit: "score" }),
+    ]);
+    const markdown = await cli(base, "--wait", WAIT_MS, "--budget-lcp", "60000", "--markdown");
+    expect(markdown.stdout, markdown.said).toMatch(/### Budgets\n\| Metric \| Value \| Budget \| Verdict \|[\s\S]*\| LCP \| [\d.]+m?s \| 60\.00s \| ✅ passed \|/);
+  }, 120000);
+});

cli/tests/e2e/matrix.test.js

@@ -152,10 +152,12 @@ describe("--matrix", () => {
   }, 120000);
 
   it("compares the budgets with the median of each combination, and names the combination", async () => {
-    const { code, stderr, said: output } = await cli("--matrix", writeMatrix(clientCase()), "--budget-lcp", "250");
+    const { code, stdout, stderr, said: output } = await cli("--matrix", writeMatrix(clientCase()), "--budget-lcp", "250");
     expect(code, output).toBe(1);
     // The phone loads the document with 300 ms of latency, so its LCP is over the budget
     expect(stderr).toMatch(/Budget violation: LCP \d+ms exceeds budget 250ms \(home · phone · anon\)/);
+    // The report says it too, under the table
+    expect(stdout).toMatch(/Budgets\n[\s\S]*home · phone · anon: LCP \d+ms exceeds budget 250ms/);
   }, 120000);
 
   it("warns about a header of the file that looks like a secret", async () => {

cli/tests/e2e/report-to.test.js

@@ -96,6 +96,14 @@ describe("--report-to", () => {
     expect(body.meta).toEqual({ viewport: "desktop", waitMs: 600, cli_version: CLI_VERSION });
   }, 30000);
 
+  it("sends the verdict of each budget, and no budgets field without them", async () => {
+    await cli(["--report-to", reportUrl, "--budget-lcp", "60000"]);
+    expect(received[0].body.budgets).toEqual([expect.objectContaining({ metric: "LCP", budget: 60000, verdict: "passed", unit: "ms" })]);
+    received.length = 0;
+    await cli(["--report-to", reportUrl]);
+    expect(received[0].body).not.toHaveProperty("budgets");
+  }, 60000);
+
   it("sends --api-key as a Bearer token", async () => {
     await cli(["--report-to", reportUrl, "--api-key", "key-from-flag"]);
     expect(received[0].headers.authorization).toBe("Bearer key-from-flag");

cli/tests/unit/reporters.test.js

@@ -612,3 +612,52 @@ describe("the human report for the Interaction snippets", () => {
     expect(output).not.toMatch(/·\s*$/m);
   });
 });
+
+describe("the budgets in the reports", () => {
+  const budgets = [
+    { metric: "LCP", budget: 2500, value: 1980, unit: "ms", verdict: "passed" },
+    { metric: "INP", budget: 200, value: 260, unit: "ms", verdict: "exceeded" },
+    { metric: "CLS", budget: 0.1, value: 0.12, unit: "score", verdict: "exceeded" },
+    { metric: "TTFB", budget: 800, value: null, unit: "ms", verdict: "not measured", note: "No navigation entry" },
+  ];
+  const withBudgets = { ...samplePayload, budgets };
+
+  it("lists every budget in the human report, passed or not, with by how much it was exceeded or why it was not measured", () => {
+    const output = reportHuman(withBudgets);
+    expect(output).toContain("Budgets");
+    expect(output).toMatch(/LCP\s+1\.98s\s+budget 2\.50s\s+passed/);
+    expect(output).toMatch(/INP\s+260ms\s+budget 200ms\s+exceeded by 60ms/);
+    expect(output).toMatch(/CLS\s+0\.1200\s+budget 0\.1000\s+exceeded by 0\.0200/);
+    expect(output).toMatch(/TTFB\s+·\s+budget 800ms\s+not measured: No navigation entry/);
+  });
+
+  it("gives the markdown report a table of the budgets, for the comment of a pull request", () => {
+    const output = reportMarkdown(withBudgets);
+    expect(output).toContain("### Budgets");
+    expect(output).toContain("| Metric | Value | Budget | Verdict |");
+    expect(output).toContain("| LCP | 1.98s | 2.50s | ✅ passed |");
+    expect(output).toContain("| INP | 260ms | 200ms | ❌ exceeded by 60ms |");
+    expect(output).toContain("| TTFB | · | 800ms | ❌ not measured: No navigation entry |");
+  });
+
+  it("adds no section without budgets", () => {
+    expect(reportHuman(samplePayload)).not.toContain("Budgets");
+    expect(reportMarkdown(samplePayload)).not.toContain("Budgets");
+  });
+
+  it("keeps the budgets in the JSON report", () => {
+    expect(JSON.parse(reportJson(withBudgets)).budgets).toEqual(budgets);
+  });
+
+  it("names the combination of each failed budget in a matrix, or says that every budget passed", () => {
+    const lcp = (value, verdict) => [{ metric: "LCP", budget: 2500, value, unit: "ms", verdict }];
+    const [first, second, failed] = matrix.combinations;
+    const failing = { ...matrix, combinations: [{ ...first, budgets: lcp(1980, "passed") }, { ...second, budgets: lcp(2710, "exceeded") }, failed] };
+    expect(reportMatrixHuman(failing)).toMatch(/Budgets[\s\S]*home · mobile · consent: LCP 2\.71s exceeds budget 2\.50s/);
+    expect(reportMatrixMarkdown(failing)).toMatch(/### Budgets\n- ❌ home · mobile · consent: LCP 2\.71s exceeds budget 2\.50s/);
+    const passing = { ...matrix, combinations: [{ ...first, budgets: lcp(1980, "passed") }, failed] };
+    expect(reportMatrixHuman(passing)).toMatch(/Budgets[\s\S]*Every budget passed/);
+    expect(reportMatrixMarkdown(passing)).toContain("✅ Every budget passed");
+    expect(reportMatrixMarkdown(matrix)).not.toContain("Budgets");
+  });
+});

content/CLI.mdx

@@ -196,7 +196,7 @@ Gate CI on performance budgets:
 npx webperf-snippets https://example.com --budget-lcp 2500 --budget-cls 0.1
 ```
 
-Each budget compares the value measured, or the median with `--runs`, and fails when the value is above it. `--budget-lcp` and `--budget-cls` need the `core-web-vitals` workflow, `--budget-fcp` the `loading` workflow, `--budget-ttfb` the `loading` or `audit` workflow, and `--budget-inp` the `interaction` workflow with a click or a key press; `--snippet` with the snippet of the metric works too. A budget of a metric that the run does not measure exits with code `2` before the page loads, and one whose metric was not measured, such as INP without interactions, exits with code `1`: a budget never passes unchecked. Each failed budget is written to stderr, also with `--json`.
+Each budget compares the value measured, or the median with `--runs`, and fails when the value is above it. `--budget-lcp` and `--budget-cls` need the `core-web-vitals` workflow, `--budget-fcp` the `loading` workflow, `--budget-ttfb` the `loading` or `audit` workflow, and `--budget-inp` the `interaction` workflow with a click or a key press; `--snippet` with the snippet of the metric works too. A budget of a metric that the run does not measure exits with code `2` before the page loads, and one whose metric was not measured, such as INP without interactions, exits with code `1`: a budget never passes unchecked. The report lists every budget with its value and verdict, `passed`, `exceeded` with by how much, or `not measured` with why: a Budgets section in the human and markdown reports, and `budgets` in the JSON output and in the body of `--report-to`. A matrix lists the failed budgets with their combination under its table. Each failed budget is also written to stderr, with `--json` too.
 
 Run the structural audit workflow: