From 3670dba7dabe88f10f6a783b85417d1563e5deed Mon Sep 17 00:00:00 2001 From: Joan Leon Date: Mon, 26 Oct 2026 00:32:04 +0100 Subject: [PATCH] feat(cli): a budget for each metric, and none that passes unchecked --budget-fcp, --budget-ttfb and --budget-inp join --budget-lcp and --budget-cls, and the interaction workflow measures INP in a step that needs a click or a key press. cli/src/budgets.js reads the budgets, knows which snippets measure each metric, and gives each budget a verdict against the value, or the median with --runs. A budget of a metric the run does not measure exits with 2 before the page loads, naming what measures it; one whose metric was not measured exits with 1; a value that is not a number exits with 2, where it passed before. stderr names the failed budgets also with --json. The GitHub Action takes budget-fcp, budget-ttfb and budget-inp, and its budget-lcp and budget-cls default to auto, their values with core-web-vitals and no budget with the workflows that do not measure LCP or CLS. --- .github/actions/webperf-snippets/action.yml | 20 +++- .github/actions/webperf-snippets/run.sh | 22 ++++- cli/CHANGELOG.md | 7 ++ cli/README.md | 16 +++- cli/src/bin.js | 70 +++++++------- cli/src/budgets.js | 73 ++++++++++++++ cli/src/workflows/interaction.js | 1 + cli/tests/e2e/budgets.test.js | 100 ++++++++++++++++++++ cli/tests/e2e/github-action.test.js | 16 ++++ cli/tests/e2e/interaction-workflow.test.js | 2 +- cli/tests/e2e/mcp-workflow.test.js | 2 +- cli/tests/unit/budgets.test.js | 97 +++++++++++++++++++ content/CLI.mdx | 19 +++- 13 files changed, 390 insertions(+), 55 deletions(-) create mode 100644 cli/src/budgets.js create mode 100644 cli/tests/e2e/budgets.test.js create mode 100644 cli/tests/unit/budgets.test.js diff --git a/.github/actions/webperf-snippets/action.yml b/.github/actions/webperf-snippets/action.yml index ee55d1f0..c988fcf7 100644 --- a/.github/actions/webperf-snippets/action.yml +++ b/.github/actions/webperf-snippets/action.yml @@ -12,11 +12,20 @@ inputs: description: Workflow to run (core-web-vitals, loading, audit, interaction) default: core-web-vitals budget-lcp: - description: LCP budget in ms. The run fails if it is exceeded. Empty disables the check. - default: "2500" + description: LCP budget in ms. The run fails if it is exceeded or LCP was not measured. "auto" is 2500 with the core-web-vitals workflow and no budget with the others, which do not measure LCP. Empty disables the check. + default: auto + budget-fcp: + description: FCP budget in ms, for the loading workflow. The run fails if it is exceeded or FCP was not measured. Empty disables the check. + default: "" + budget-ttfb: + description: TTFB budget in ms, for the loading and audit workflows. The run fails if it is exceeded or TTFB was not measured. Empty disables the check. + default: "" budget-cls: - description: CLS budget. The run fails if it is exceeded. Empty disables the check. - default: "0.1" + description: CLS budget. The run fails if it is exceeded or CLS was not measured. "auto" is 0.1 with the core-web-vitals workflow and no budget with the others, which do not measure CLS. Empty disables the check. + default: auto + budget-inp: + description: INP budget in ms, for the interaction workflow with clicks or key presses. The run fails if it is exceeded or INP was not measured. Empty disables the check. + default: "" fail-on-budget: description: Fail the step when a budget is exceeded or a snippet errors. Set to "false" to read the exit-code output and decide yourself, for example to post the markdown first. default: "true" @@ -53,6 +62,9 @@ runs: WEBPERF_URL: ${{ inputs.url }} WEBPERF_WORKFLOW: ${{ inputs.workflow }} WEBPERF_BUDGET_LCP: ${{ inputs.budget-lcp }} + WEBPERF_BUDGET_FCP: ${{ inputs.budget-fcp }} + WEBPERF_BUDGET_TTFB: ${{ inputs.budget-ttfb }} WEBPERF_BUDGET_CLS: ${{ inputs.budget-cls }} + WEBPERF_BUDGET_INP: ${{ inputs.budget-inp }} WEBPERF_FAIL_ON_BUDGET: ${{ inputs.fail-on-budget }} run: bash "${{ github.action_path }}/run.sh" diff --git a/.github/actions/webperf-snippets/run.sh b/.github/actions/webperf-snippets/run.sh index 2ca1a0d6..60810d2e 100755 --- a/.github/actions/webperf-snippets/run.sh +++ b/.github/actions/webperf-snippets/run.sh @@ -11,9 +11,25 @@ set -uo pipefail read -r -a cmd <<< "${WEBPERF_CMD:?WEBPERF_CMD is required}" -args=("${WEBPERF_URL:?WEBPERF_URL is required}" --workflow "${WEBPERF_WORKFLOW:-core-web-vitals}" --markdown) -[ -n "${WEBPERF_BUDGET_LCP:-}" ] && args+=(--budget-lcp "$WEBPERF_BUDGET_LCP") -[ -n "${WEBPERF_BUDGET_CLS:-}" ] && args+=(--budget-cls "$WEBPERF_BUDGET_CLS") +workflow="${WEBPERF_WORKFLOW:-core-web-vitals}" +args=("${WEBPERF_URL:?WEBPERF_URL is required}" --workflow "$workflow" --markdown) + +# Adds a budget flag when its input has a value. "auto", the default of LCP and CLS, means the default +# value with the core-web-vitals workflow, the one that measures them, and no budget with the others: +# the CLI rejects a budget of a metric the workflow does not measure. +budget() { + local option=$1 value=$2 default=${3:-} + if [ "$value" = "auto" ]; then + [ "$workflow" = "core-web-vitals" ] && value=$default || value="" + fi + [ -n "$value" ] && args+=("$option" "$value") + return 0 +} +budget --budget-lcp "${WEBPERF_BUDGET_LCP:-}" 2500 +budget --budget-fcp "${WEBPERF_BUDGET_FCP:-}" +budget --budget-ttfb "${WEBPERF_BUDGET_TTFB:-}" +budget --budget-cls "${WEBPERF_BUDGET_CLS:-}" 0.1 +budget --budget-inp "${WEBPERF_BUDGET_INP:-}" markdown=$("${cmd[@]}" "${args[@]}") code=$? diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 27428315..ce684263 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -8,6 +8,13 @@ Format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). The CL ## [Unreleased] +### Added +- `--budget-fcp`, `--budget-ttfb` and `--budget-inp`, next to `--budget-lcp` and `--budget-cls`: a budget for each metric the CLI measures. The interaction workflow measures INP, in a step `INP` that needs a click or a key press like `interactions` and `input-latency`. The GitHub Action takes `budget-fcp`, `budget-ttfb` and `budget-inp`. + +### Changed +- A budget never passes unchecked. One whose metric the run does not measure, such as `--budget-lcp` with `--workflow audit`, exits with `2` before the page loads, naming the workflow or snippet that measures it; one whose metric was not measured, such as INP without interactions or an LCP that failed, exits with `1`. A value that is not a number of 0 or more exits with `2`; it passed before. Each failed budget goes to stderr also with `--json`. +- The `budget-lcp` and `budget-cls` inputs of the GitHub Action default to `auto`: 2500 and 0.1 with the `core-web-vitals` workflow, and no budget with the others, which do not measure LCP or CLS. An empty value still disables the check. + --- ## [1.5.0] — 2026-10-25 diff --git a/cli/README.md b/cli/README.md index 981992f0..d8ea8652 100644 --- a/cli/README.md +++ b/cli/README.md @@ -300,6 +300,8 @@ CI gating: npx webperf-snippets https://example.com --budget-lcp 2500 --budget-cls 0.1 ``` +Each budget compares the value measured, or the median with `--runs`, and fails when the value is above it. `--budget-lcp` and `--budget-cls` need the `core-web-vitals` workflow, `--budget-fcp` the `loading` workflow, `--budget-ttfb` the `loading` or `audit` workflow, and `--budget-inp` the `interaction` workflow with a click or a key press; `--snippet` with the snippet of the metric works too. A budget of a metric that the run does not measure exits with code `2` before the page loads, and one whose metric was not measured, such as INP without interactions, exits with code `1`: a budget never passes unchecked. Each failed budget is written to stderr, also with `--json`. + ### Options | Option | Description | @@ -325,8 +327,11 @@ npx webperf-snippets https://example.com --budget-lcp 2500 --budget-cls 0.1 | `--crux-key ` | Add CrUX field data (p75) next to the measured values. Prefer `CRUX_API_KEY`. | | `--report-to ` | POST the results to this `https` URL after the run. No external calls without it. | | `--api-key ` | Sent as a Bearer token with `--report-to`. Prefer `PERF_REVIEWS_API_KEY`. | -| `--budget-lcp ` | Exit `1` if LCP exceeds this value. | -| `--budget-cls ` | Exit `1` if CLS exceeds this value. | +| `--budget-lcp ` | Exit `1` if LCP exceeds this value or was not measured. | +| `--budget-fcp ` | Exit `1` if FCP exceeds this value or was not measured. | +| `--budget-ttfb ` | Exit `1` if TTFB exceeds this value or was not measured. | +| `--budget-cls ` | Exit `1` if CLS exceeds this value or was not measured. | +| `--budget-inp ` | Exit `1` if INP exceeds this value or was not measured. | | `--verbose` | Show all items, including passing checks. | | `--headed` | Show the browser window (debug). | | `-h, --help` | Show help. | @@ -453,8 +458,11 @@ jobs: | ---------------- | ------------------- | -------------------------------------------------------------------------------------------- | | `url` | required | URL to measure. | | `workflow` | `core-web-vitals` | Workflow to run: `core-web-vitals`, `loading`, `audit` or `interaction`. | -| `budget-lcp` | `2500` | LCP budget in ms. An empty value disables the check. | -| `budget-cls` | `0.1` | CLS budget. An empty value disables the check. | +| `budget-lcp` | `auto` | LCP budget in ms. `auto` is `2500` with `core-web-vitals` and no budget with the other workflows. An empty value disables the check. | +| `budget-fcp` | | FCP budget in ms, for `loading`. | +| `budget-ttfb` | | TTFB budget in ms, for `loading` or `audit`. | +| `budget-cls` | `auto` | CLS budget. `auto` is `0.1` with `core-web-vitals` and no budget with the other workflows. An empty value disables the check. | +| `budget-inp` | | INP budget in ms, for `interaction` with a click or a key press. | | `fail-on-budget` | `true` | Fail the step when a budget is exceeded or a snippet errors. Use `"false"` to read `exit-code` and decide yourself. | | `cli-version` | `latest` | Version of the `webperf-snippets` npm package to run. | diff --git a/cli/src/bin.js b/cli/src/bin.js index 9d125e0d..6e8b890b 100755 --- a/cli/src/bin.js +++ b/cli/src/bin.js @@ -18,6 +18,7 @@ import { parseOwnDomains } from "./own-domains.js"; import { MEDIAN_METRICS, summarizeRuns } from "./runs.js"; import { loadMatrix } from "./matrix.js"; import { blockingErrors } from "./blocking.js"; +import { MEASURED_BY, checkBudgets, parseBudgets, unmeasuredMetrics, violationsOf } from "./budgets.js"; import { measurementConditions } from "./conditions.js"; import { loadBaseline, parseThreshold, thresholdsWith, compareRun, compareMatrix } from "./baseline.js"; import { describeRegressions, describeMatrixRegressions, matrixComparisonView } from "./reporters/baseline-view.js"; @@ -49,8 +50,11 @@ Options: --wait Post-load wait before evaluating (default: 3000) --cpu Slow the CPU down by this factor, such as 4 (no throttling by default) --network Throttle the network as DevTools does: fast-4g, slow-4g, 3g - --budget-lcp Exit 1 if LCP exceeds this value - --budget-cls Exit 1 if CLS exceeds this value + --budget-lcp Exit 1 if LCP exceeds this value, or was not measured + --budget-fcp Exit 1 if FCP exceeds this value, or was not measured + --budget-ttfb Exit 1 if TTFB exceeds this value, or was not measured + --budget-cls Exit 1 if CLS exceeds this value, or was not measured + --budget-inp Exit 1 if INP exceeds this value, or was not measured (interaction workflow) --interact-script JSON file with interactions to run after the load; the Interaction snippets are installed first and read last Actions: scroll, click, hover, type, wait @@ -124,30 +128,13 @@ function parseRunCount(raw) { return /^[1-9]$/.test(raw.trim()) ? Number(raw) : null; } -function checkBudgets({ results, runs }, values) { - const violations = []; - const lcpBudget = values["budget-lcp"] ? Number(values["budget-lcp"]) : null; - const clsBudget = values["budget-cls"] ? Number(values["budget-cls"]) : null; - - // With several runs the budgets compare the median of each metric, not one sample - if (runs) { - const lcp = runs.summary.LCP?.median; - const cls = runs.summary.CLS?.median; - if (lcp != null && lcpBudget != null && lcp > lcpBudget) violations.push(`LCP ${lcp}ms exceeds budget ${lcpBudget}ms`); - if (cls != null && clsBudget != null && cls > clsBudget) violations.push(`CLS ${cls} exceeds budget ${clsBudget}`); - return violations; - } - - for (const r of results) { - if (r.status !== "ok") continue; - if (r.metric === "LCP" && lcpBudget != null && r.value > lcpBudget) { - violations.push(`LCP ${r.value}ms exceeds budget ${lcpBudget}ms`); - } - if (r.metric === "CLS" && clsBudget != null && r.value > clsBudget) { - violations.push(`CLS ${r.value} exceeds budget ${clsBudget}`); - } - } - return violations; +// What to run instead, when a budget names a metric the run does not measure +function budgetUsage(metric, run) { + const workflows = Object.values(WORKFLOWS) + .filter((w) => w.steps.some((step) => MEASURED_BY[metric].includes(step.path))) + .map((w) => w.id); + const instead = [...workflows.map((id) => `--workflow ${id}`), `--snippet ${MEASURED_BY[metric][0].split("/").pop()}`].join(" or "); + return `--budget-${metric.toLowerCase()} needs a run that measures ${metric}, and ${run} does not. Use ${instead}`; } async function main() { @@ -168,7 +155,10 @@ async function main() { markdown: { type: "boolean" }, wait: { type: "string" }, "budget-lcp": { type: "string" }, + "budget-fcp": { type: "string" }, + "budget-ttfb": { type: "string" }, "budget-cls": { type: "string" }, + "budget-inp": { type: "string" }, viewport: { type: "string" }, cpu: { type: "string" }, network: { type: "string" }, @@ -275,10 +265,20 @@ async function main() { if (values["fail-on-regression"]) fail("--fail-on-regression needs --baseline"); } + let budgets; + try { + budgets = parseBudgets(values); + } catch (err) { + fail(err.message); + } + // What each page load runs, a snippet or a workflow, under the conditions of the load const items = values.snippet ? buildSnippetItem(values) : null; const workflow = values.snippet ? null : WORKFLOWS[values.workflow ?? "core-web-vitals"]; if (!values.snippet && !workflow) fail(`Unknown workflow: ${values.workflow}`); + // A budget of a metric that no snippet of the run measures could never pass: a usage error + const [unmeasured] = unmeasuredMetrics(budgets, items ? [items[0].path] : workflow.steps.map((step) => step.path)); + if (unmeasured) fail(budgetUsage(unmeasured, items ? `--snippet ${values.snippet}` : `the ${workflow.id} workflow`)); const measureWith = (conditions) => { const options = { waitMs, headless: !values.headed, interactScript, ...conditions }; return items ? runSnippets({ ...options, items }) : runMeasurement({ ...options, workflow, rules: RULES }); @@ -294,7 +294,7 @@ async function main() { } catch (err) { fail(err.message); } - await runMatrix(matrixPath, matrix, measureWith, { values, waitMs, reportUrl, conditionsFor, baseline, thresholds }); + await runMatrix(matrixPath, matrix, measureWith, { values, waitMs, reportUrl, conditionsFor, baseline, thresholds, budgets }); return; } @@ -349,12 +349,10 @@ async function main() { } } - // Exit codes. - const violations = checkBudgets(payload, values); + // Exit codes. stderr names the budgets that failed, also with --json, whose report is on stdout + const violations = violationsOf(checkBudgets(payload, budgets)); if (violations.length > 0) { - if (!values.json) { - for (const v of violations) process.stderr.write(`Budget violation: ${v}\n`); - } + for (const v of violations) process.stderr.write(`Budget violation: ${v}\n`); process.exit(1); } // Without --fail-on-regression the comparison only informs, and the exit code is the one of today @@ -385,7 +383,7 @@ async function measureRuns(measure, count, medianBy, progress) { // Every combination of the matrix, in order, each one `matrix.runs` times. Each combination reports // its median run, and the exit code follows the same rules as one page, for every combination. -async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, reportUrl, conditionsFor, baseline, thresholds }) { +async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, reportUrl, conditionsFor, baseline, thresholds, budgets }) { for (const warning of matrix.warnings) process.stderr.write(`Warning: ${warning}\n`); const viewportOf = new Map(matrix.combinations.map((c) => [c.id, c.viewport])); @@ -464,10 +462,8 @@ async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, repo } } - const violations = combinations.filter((c) => !c.error).flatMap((c) => checkBudgets(c, values).map((v) => `${v} (${c.id})`)); - if (!values.json) { - for (const v of violations) process.stderr.write(`Budget violation: ${v}\n`); - } + const violations = combinations.filter((c) => !c.error).flatMap((c) => violationsOf(checkBudgets(c, budgets)).map((v) => `${v} (${c.id})`)); + for (const v of violations) process.stderr.write(`Budget violation: ${v}\n`); const failed = combinations.some((c) => c.error || blockingErrors(c.results).length > 0); // Without --fail-on-regression the comparison only informs const regressions = values["fail-on-regression"] ? describeMatrixRegressions(combinations) : []; diff --git a/cli/src/budgets.js b/cli/src/budgets.js new file mode 100644 index 00000000..61153c9b --- /dev/null +++ b/cli/src/budgets.js @@ -0,0 +1,73 @@ +// The performance budgets of the command line (--budget-lcp, --budget-fcp, --budget-ttfb, +// --budget-cls and --budget-inp): reading them, telling whether the run can measure their metric, +// and the verdict of each one against what a page measured. + +// The metrics that take a budget, with their unit +export const BUDGET_METRICS = { LCP: "ms", FCP: "ms", TTFB: "ms", CLS: "score", INP: "ms" }; + +// The snippets whose result carries each metric, by path +export const MEASURED_BY = { + LCP: ["CoreWebVitals/LCP", "CoreWebVitals/LCP-Subparts"], + FCP: ["Loading/FCP"], + TTFB: ["Loading/TTFB", "Loading/TTFB-Sub-Parts"], + CLS: ["CoreWebVitals/CLS"], + INP: ["CoreWebVitals/INP"], +}; + +const optionOf = (metric) => `--budget-${metric.toLowerCase()}`; + +// The budgets given on the command line, by metric. A value that is not a number of 0 or more throws, +// so the command can exit with 2 before the browser opens. +export function parseBudgets(values) { + const budgets = {}; + for (const metric of Object.keys(BUDGET_METRICS)) { + const raw = values[optionOf(metric).slice(2)]; + if (raw === undefined) continue; + if (!/^\d+(?:\.\d+)?$/.test(raw.trim())) { + throw new Error(`Invalid ${optionOf(metric)} "${raw}". Use a number of 0 or more, such as ${metric === "CLS" ? "0.1" : "2500"}`); + } + budgets[metric] = Number(raw); + } + return budgets; +} + +// The metrics with a budget that no snippet of the run measures, such as LCP in the audit workflow +export const unmeasuredMetrics = (budgets, paths) => + Object.keys(budgets).filter((metric) => !MEASURED_BY[metric].some((path) => paths.includes(path))); + +const nameOf = (path) => path.split("/").pop().toLowerCase(); + +// The result of the snippet that measures a metric, a step of the run before a follow-up +function resultFor(results, metric) { + const names = MEASURED_BY[metric].map(nameOf); + const ofMetric = results.filter((r) => r.metric === metric || names.includes(String(r.script ?? r.id).toLowerCase())); + return ofMetric.find((r) => !r.reason || r.status === "skipped") ?? ofMetric[0]; +} + +// The verdict of each budget: passed when the value is at most the budget, exceeded above it, and not +// measured when the run has no value, with the reason its snippet gave. With several runs, the median. +export function checkBudgets({ results = [], runs }, budgets) { + return Object.entries(budgets).map(([metric, budget]) => { + const unit = BUDGET_METRICS[metric]; + const result = resultFor(results, metric); + const measured = result?.status === "ok" && typeof result.value === "number" ? result.value : null; + const value = runs ? (runs.summary?.[metric]?.median ?? null) : measured; + if (value === null) { + const note = result?.error ?? result?.message ?? (result?.status === "skipped" ? result.reason : undefined); + return { metric, budget, value: null, unit, verdict: "not measured", ...(note ? { note } : {}) }; + } + return { metric, budget, value, unit, verdict: value > budget ? "exceeded" : "passed" }; + }); +} + +const amount = (value, unit) => (unit === "ms" ? `${value}ms` : String(value)); + +// The budgets that failed, in the words of stderr +export const violationsOf = (checked) => + checked + .filter((b) => b.verdict !== "passed") + .map((b) => + b.verdict === "exceeded" + ? `${b.metric} ${amount(b.value, b.unit)} exceeds budget ${amount(b.budget, b.unit)}` + : `${b.metric} was not measured (budget ${amount(b.budget, b.unit)})${b.note ? `: ${b.note}` : ""}` + ); diff --git a/cli/src/workflows/interaction.js b/cli/src/workflows/interaction.js index be65db71..ccab87d1 100644 --- a/cli/src/workflows/interaction.js +++ b/cli/src/workflows/interaction.js @@ -24,6 +24,7 @@ export const interactionWorkflow = { { id: "scroll", path: "Interaction/Scroll-Performance" }, { id: "layout-shifts", path: "Interaction/Layout-Shift-Loading-and-Interaction" }, { id: "forced-layout", path: "Interaction/Forced-Synchronous-Layout" }, + { id: "INP", path: "CoreWebVitals/INP", needsInput: true }, { id: "interactions", path: "Interaction/Interactions", needsInput: true }, { id: "input-latency", path: "Interaction/Input-Latency-Breakdown", needsInput: true }, ], diff --git a/cli/tests/e2e/budgets.test.js b/cli/tests/e2e/budgets.test.js new file mode 100644 index 00000000..d50fb698 --- /dev/null +++ b/cli/tests/e2e/budgets.test.js @@ -0,0 +1,100 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { createServer } from "node:http"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import { cliEnv } from "../helpers/cli-env.js"; + +const BIN = join(dirname(fileURLToPath(import.meta.url)), "../../src/bin.js"); +const run = promisify(execFile); +// The LCP entry needs a moment after load to be reported, as the other tests of the CLI note +const WAIT_MS = "500"; +const said = (stdout = "", stderr = "") => `stderr:\n${stderr}\nstdout:\n${stdout.slice(0, 3000)}`; + +// A page with a heading for LCP and a button whose click handler blocks the main thread for 300 ms, +// far above an INP budget of 200 ms. Its async script, cached for a year, gives the snippets of the +// loading workflow a resource to read. The server counts its requests: a run that fails on its +// options makes none. +let server; +let base; +let dir; +let requests = 0; +const PAGE = `Budgets

A page with a slow button

+ + +`; + +beforeAll(async () => { + server = await new Promise((resolve) => { + const s = createServer((req, res) => { + requests++; + if (req.url === "/app.js") return res.writeHead(200, { "Content-Type": "text/javascript", "Cache-Control": "public, max-age=31536000, immutable" }).end("1;"); + res.writeHead(200, { "Content-Type": "text/html" }).end(PAGE); + }); + s.listen(0, "127.0.0.1", () => resolve(s)); + }); + base = `http://127.0.0.1:${server.address().port}/`; + dir = mkdtempSync(join(tmpdir(), "webperf-budgets-")); +}); + +afterAll(() => new Promise((resolve) => server.close(resolve))); + +const cli = (...args) => + run("node", [BIN, ...args], { env: cliEnv() }).then( + ({ stdout, stderr }) => ({ code: 0, stdout, stderr, said: said(stdout, stderr) }), + (err) => ({ code: err.code, stdout: err.stdout, stderr: err.stderr, said: said(err.stdout, err.stderr) }) + ); + +const steps = (name, interactions) => { + const file = join(dir, name); + writeFileSync(file, JSON.stringify({ interactions })); + return file; +}; + +describe("--budget-inp", () => { + it("exceeds the budget with a slow click, in the interaction workflow", async () => { + const clicks = steps("click.json", [{ action: "click", selector: "#slow" }, { action: "wait", ms: 300 }]); + const { code, stderr, said: output } = await cli(base, "--workflow", "interaction", "--interact-script", clicks, "--budget-inp", "200"); + expect(code, output).toBe(1); + expect(stderr).toMatch(/Budget violation: INP \d+ms exceeds budget 200ms/); + }, 120000); + + it("fails as not measured without a click, instead of passing", async () => { + const { code, stderr, said: output } = await cli(base, "--workflow", "interaction", "--budget-inp", "200"); + expect(code, output).toBe(1); + expect(stderr).toMatch(/Budget violation: INP was not measured \(budget 200ms\)/); + }, 120000); +}); + +describe("budgets that cannot be checked", () => { + it("exits with 2 before the browser opens for a metric the run does not measure", async () => { + const before = requests; + const { code, stderr } = await cli(base, "--workflow", "audit", "--budget-lcp", "2500"); + expect(code).toBe(2); + expect(stderr).toMatch(/--budget-lcp needs a run that measures LCP/); + expect(requests).toBe(before); + }, 30000); + + it("exits with 2 before the browser opens for a value that is not a number", async () => { + const before = requests; + const { code, stderr } = await cli(base, "--budget-fcp", "fast"); + expect(code).toBe(2); + expect(stderr).toMatch(/Invalid --budget-fcp "fast"/); + expect(requests).toBe(before); + }, 30000); +}); + +describe("--budget-fcp and --budget-ttfb", () => { + it("pass with a loose budget and fail with an impossible one, in the loading workflow", async () => { + const loose = await cli(base, "--workflow", "loading", "--wait", WAIT_MS, "--budget-fcp", "60000", "--budget-ttfb", "60000"); + expect(loose.code, loose.said).toBe(0); + const tight = await cli(base, "--workflow", "loading", "--wait", WAIT_MS, "--budget-fcp", "0", "--budget-ttfb", "0", "--json"); + expect(tight.code, tight.said).toBe(1); + // stderr names the violations also with --json, since stdout carries the JSON + expect(tight.stderr).toMatch(/Budget violation: FCP \d+ms exceeds budget 0ms/); + expect(tight.stderr).toMatch(/Budget violation: TTFB [\d.]+ms exceeds budget 0ms/); + }, 120000); +}); diff --git a/cli/tests/e2e/github-action.test.js b/cli/tests/e2e/github-action.test.js index a753353a..6e25f398 100644 --- a/cli/tests/e2e/github-action.test.js +++ b/cli/tests/e2e/github-action.test.js @@ -170,6 +170,22 @@ describe("run.sh arguments", () => { ]); }); + it("forwards the budgets of FCP, TTFB and INP", async () => { + const none = { WEBPERF_BUDGET_LCP: "", WEBPERF_BUDGET_CLS: "" }; + await runAction({ ...none, WEBPERF_WORKFLOW: "loading", WEBPERF_BUDGET_FCP: "1800", WEBPERF_BUDGET_TTFB: "800" }); + expect(argsSeen()).toEqual(["https://example.com", "--workflow", "loading", "--markdown", "--budget-fcp", "1800", "--budget-ttfb", "800"]); + await runAction({ ...none, WEBPERF_WORKFLOW: "interaction", WEBPERF_BUDGET_INP: "200" }); + expect(argsSeen()).toEqual(["https://example.com", "--workflow", "interaction", "--markdown", "--budget-inp", "200"]); + }); + + it("applies the default budgets of LCP and CLS (auto) to the core-web-vitals workflow only", async () => { + await runAction({ WEBPERF_BUDGET_LCP: "auto", WEBPERF_BUDGET_CLS: "auto" }); + expect(argsSeen()).toEqual(["https://example.com", "--workflow", "core-web-vitals", "--markdown", "--budget-lcp", "2500", "--budget-cls", "0.1"]); + // The audit workflow measures neither, so the defaults would be a usage error + await runAction({ WEBPERF_WORKFLOW: "audit", WEBPERF_BUDGET_LCP: "auto", WEBPERF_BUDGET_CLS: "auto" }); + expect(argsSeen()).toEqual(["https://example.com", "--workflow", "audit", "--markdown"]); + }); + it("omits a budget flag when its input is empty", async () => { await runAction({ WEBPERF_BUDGET_LCP: "", WEBPERF_BUDGET_CLS: "" }); expect(argsSeen()).toEqual(["https://example.com", "--workflow", "core-web-vitals", "--markdown"]); diff --git a/cli/tests/e2e/interaction-workflow.test.js b/cli/tests/e2e/interaction-workflow.test.js index a31a9851..fd454c56 100644 --- a/cli/tests/e2e/interaction-workflow.test.js +++ b/cli/tests/e2e/interaction-workflow.test.js @@ -44,7 +44,7 @@ beforeAll( afterAll(() => new Promise((resolve) => server.close(resolve))); -const NEEDS_INPUT = ["interactions", "input-latency"]; +const NEEDS_INPUT = ["INP", "interactions", "input-latency"]; const CLICK = [{ action: "click", selector: "#btn" }, { action: "wait", ms: 400 }]; async function measure(interactions) { diff --git a/cli/tests/e2e/mcp-workflow.test.js b/cli/tests/e2e/mcp-workflow.test.js index 0783be39..b90f44de 100644 --- a/cli/tests/e2e/mcp-workflow.test.js +++ b/cli/tests/e2e/mcp-workflow.test.js @@ -95,7 +95,7 @@ describe("run_workflow", () => { await connect(); const result = await call("run_workflow", { url: `${servers.base}/seeded`, workflow: "interaction", waitMs: 0 }); const skipped = result.structuredContent.results.filter((r) => r.status === "skipped"); - expect(skipped.map((r) => r.id)).toEqual(["interactions", "input-latency"]); + expect(skipped.map((r) => r.id)).toEqual(["INP", "interactions", "input-latency"]); for (const r of skipped) { expect(r.reason).toMatch(/open_page/); expect(r.reason).not.toMatch(/--interact-script/); diff --git a/cli/tests/unit/budgets.test.js b/cli/tests/unit/budgets.test.js new file mode 100644 index 00000000..ca5406da --- /dev/null +++ b/cli/tests/unit/budgets.test.js @@ -0,0 +1,97 @@ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BUDGET_METRICS, MEASURED_BY, parseBudgets, unmeasuredMetrics, checkBudgets, violationsOf } from "../../src/budgets.js"; +import { WORKFLOWS } from "../../src/workflows/index.js"; + +const SNIPPETS = join(dirname(fileURLToPath(import.meta.url)), "../../../snippets"); + +describe("parseBudgets", () => { + it("reads a budget for each of the five metrics, and leaves out the ones not given", () => { + expect(parseBudgets({ "budget-lcp": "2500", "budget-cls": "0.1", "budget-inp": "200", "budget-fcp": "1800", "budget-ttfb": "800" })).toEqual({ + LCP: 2500, + CLS: 0.1, + INP: 200, + FCP: 1800, + TTFB: 800, + }); + expect(parseBudgets({ "budget-lcp": "2500" })).toEqual({ LCP: 2500 }); + expect(parseBudgets({})).toEqual({}); + }); + + it("rejects a value that is not a number of 0 or more, naming the option", () => { + for (const raw of ["abc", "-1", "", "2500ms", "1e"]) { + expect(() => parseBudgets({ "budget-lcp": raw })).toThrow(/--budget-lcp/); + } + expect(parseBudgets({ "budget-cls": "0" })).toEqual({ CLS: 0 }); + }); +}); + +describe("MEASURED_BY", () => { + it("names, for each metric, the snippets whose result carries it", () => { + expect(Object.keys(MEASURED_BY).sort()).toEqual(Object.keys(BUDGET_METRICS).sort()); + for (const [metric, paths] of Object.entries(MEASURED_BY)) { + for (const path of paths) expect(readFileSync(join(SNIPPETS, `${path}.js`), "utf8")).toContain(`metric: "${metric}"`); + } + }); + + it("makes the interaction workflow measure INP", () => { + expect(WORKFLOWS.interaction.steps.map((s) => s.path)).toContain("CoreWebVitals/INP"); + }); +}); + +describe("unmeasuredMetrics", () => { + it("lists the budgets whose metric no snippet of the run measures", () => { + const audit = WORKFLOWS.audit.steps.map((s) => s.path); + expect(unmeasuredMetrics({ LCP: 2500, CLS: 0.1, TTFB: 800 }, audit)).toEqual(["LCP", "CLS"]); + expect(unmeasuredMetrics({ LCP: 2500, CLS: 0.1 }, WORKFLOWS["core-web-vitals"].steps.map((s) => s.path))).toEqual([]); + expect(unmeasuredMetrics({ INP: 200 }, ["CoreWebVitals/INP"])).toEqual([]); + }); +}); + +describe("checkBudgets", () => { + const page = (results, runs) => ({ results, ...(runs ? { runs } : {}) }); + const metric = (name, value, unit = "ms", extra = {}) => ({ id: name, script: name, status: "ok", metric: name, value, unit, ...extra }); + + it("passes a value equal to the budget, and exceeds one above it", () => { + expect(checkBudgets(page([metric("LCP", 2500), metric("CLS", 0.12, "score")]), { LCP: 2500, CLS: 0.1 })).toEqual([ + { metric: "LCP", budget: 2500, value: 2500, unit: "ms", verdict: "passed" }, + { metric: "CLS", budget: 0.1, value: 0.12, unit: "score", verdict: "exceeded" }, + ]); + }); + + it("says why a metric was not measured, from the result of its snippet", () => { + const inp = { id: "INP", script: "INP", status: "tracking", message: "No interactions yet: click, tap or type on the page" }; + const lcp = { id: "LCP", status: "error", error: "No LCP entries buffered" }; + expect(checkBudgets(page([inp, lcp]), { INP: 200, LCP: 2500 })).toEqual([ + { metric: "INP", budget: 200, value: null, unit: "ms", verdict: "not measured", note: "No interactions yet: click, tap or type on the page" }, + { metric: "LCP", budget: 2500, value: null, unit: "ms", verdict: "not measured", note: "No LCP entries buffered" }, + ]); + const skipped = { id: "INP", status: "skipped", reason: "The interactions have no click or type step" }; + expect(checkBudgets(page([skipped]), { INP: 200 })[0]).toEqual(expect.objectContaining({ verdict: "not measured", note: "The interactions have no click or type step" })); + }); + + it("takes the value of the step before the one of a follow-up", () => { + const followUp = metric("LCP", 900, "ms", { id: "lcp-subparts", script: "LCP-Subparts", reason: "LCP is poor" }); + expect(checkBudgets(page([followUp, metric("LCP", 3100)]), { LCP: 2500 })[0]).toEqual(expect.objectContaining({ value: 3100, verdict: "exceeded" })); + }); + + it("compares the median of the runs", () => { + const runs = { count: 3, summary: { LCP: { median: 2400, min: 2100, max: 3000, unit: "ms" } } }; + expect(checkBudgets(page([metric("LCP", 3000)], runs), { LCP: 2500 })[0]).toEqual(expect.objectContaining({ value: 2400, verdict: "passed" })); + expect(checkBudgets(page([], { count: 3, summary: {} }), { INP: 200 })[0]).toEqual(expect.objectContaining({ value: null, verdict: "not measured" })); + }); +}); + +describe("violationsOf", () => { + it("words each budget that failed, as stderr says it", () => { + const checked = [ + { metric: "LCP", budget: 2500, value: 3100, unit: "ms", verdict: "exceeded" }, + { metric: "CLS", budget: 0.1, value: 0.05, unit: "score", verdict: "passed" }, + { metric: "CLS", budget: 0.1, value: 0.3, unit: "score", verdict: "exceeded" }, + { metric: "INP", budget: 200, value: null, unit: "ms", verdict: "not measured", note: "No interactions yet" }, + ]; + expect(violationsOf(checked)).toEqual(["LCP 3100ms exceeds budget 2500ms", "CLS 0.3 exceeds budget 0.1", "INP was not measured (budget 200ms): No interactions yet"]); + }); +}); diff --git a/content/CLI.mdx b/content/CLI.mdx index d1413e11..eda574e6 100644 --- a/content/CLI.mdx +++ b/content/CLI.mdx @@ -196,6 +196,8 @@ Gate CI on performance budgets: npx webperf-snippets https://example.com --budget-lcp 2500 --budget-cls 0.1 ``` +Each budget compares the value measured, or the median with `--runs`, and fails when the value is above it. `--budget-lcp` and `--budget-cls` need the `core-web-vitals` workflow, `--budget-fcp` the `loading` workflow, `--budget-ttfb` the `loading` or `audit` workflow, and `--budget-inp` the `interaction` workflow with a click or a key press; `--snippet` with the snippet of the metric works too. A budget of a metric that the run does not measure exits with code `2` before the page loads, and one whose metric was not measured, such as INP without interactions, exits with code `1`: a budget never passes unchecked. Each failed budget is written to stderr, also with `--json`. + Run the structural audit workflow: ```bash copy @@ -279,10 +281,11 @@ It installs the Interaction snippets, runs the interactions, and then reads them | `scroll` | Scroll frame rate, non-passive listeners and CSS that affects scrolling | | `layout-shifts` | Layout shifts, split between loading and interaction | | `forced-layout` | Forced synchronous layouts | +| `INP` | Interaction to Next Paint, the worst interaction with its phases | | `interactions` | Every interaction with its latency and rating | | `input-latency` | Input delay, processing time and presentation delay of each event type | -`interactions` and `input-latency` need a click or a `type` step. When the interactions have none, they appear as `skipped` with the reason, and they do not count as errors. When the script has a click but the page records no interaction, they report an error, which tells you the step did not reach an element that responds. +`INP`, `interactions` and `input-latency` need a click or a `type` step. When the interactions have none, they appear as `skipped` with the reason, and they do not count as errors. When the script has a click but the page records no interaction, they report an error, which tells you the step did not reach an element that responds. Simulate a mobile viewport (default) or switch to desktop: @@ -447,8 +450,11 @@ npx webperf-snippets https://example.com --verbose | `--crux-key ` | Add CrUX field data (p75) next to the measured values. Prefer `CRUX_API_KEY`. | | `--report-to ` | POST the results to this `https` URL after the run. No external calls without it. | | `--api-key ` | Sent as a Bearer token with `--report-to`. Prefer `PERF_REVIEWS_API_KEY`. | -| `--budget-lcp ` | Exit `1` if LCP exceeds this value. | -| `--budget-cls ` | Exit `1` if CLS exceeds this value. | +| `--budget-lcp ` | Exit `1` if LCP exceeds this value or was not measured. | +| `--budget-fcp ` | Exit `1` if FCP exceeds this value or was not measured. | +| `--budget-ttfb ` | Exit `1` if TTFB exceeds this value or was not measured. | +| `--budget-cls ` | Exit `1` if CLS exceeds this value or was not measured. | +| `--budget-inp ` | Exit `1` if INP exceeds this value or was not measured. | | `--verbose` | Show all items, including passing checks. | | `--headed` | Show the browser window (debug). | | `-h, --help` | Show help. | @@ -534,8 +540,11 @@ jobs: | ---------------- | ------------------- | -------------------------------------------------------------------------------------------- | | `url` | required | URL to measure. | | `workflow` | `core-web-vitals` | Workflow to run: `core-web-vitals`, `loading`, `audit` or `interaction`. | -| `budget-lcp` | `2500` | LCP budget in ms. An empty value disables the check. | -| `budget-cls` | `0.1` | CLS budget. An empty value disables the check. | +| `budget-lcp` | `auto` | LCP budget in ms. `auto` is `2500` with `core-web-vitals` and no budget with the other workflows. An empty value disables the check. | +| `budget-fcp` | | FCP budget in ms, for `loading`. | +| `budget-ttfb` | | TTFB budget in ms, for `loading` or `audit`. | +| `budget-cls` | `auto` | CLS budget. `auto` is `0.1` with `core-web-vitals` and no budget with the other workflows. An empty value disables the check. | +| `budget-inp` | | INP budget in ms, for `interaction` with a click or a key press. | | `fail-on-budget` | `true` | Fail the step when a budget is exceeded or a snippet errors. Use `"false"` to read `exit-code` and decide yourself. | | `cli-version` | `latest` | Version of the `webperf-snippets` npm package to run. |