From 043430ac9eb043e4c6affc4464972fc2240742e4 Mon Sep 17 00:00:00 2001 From: "user.email" <136036952+easonLiangWorldedtech@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:03:53 +0800 Subject: [PATCH 1/3] scripts: name the mutation gate's budget, binary, and rejection record The gate handed Stryker no initial-test-run budget, so the engine's own schema default of five minutes decided when an initial test run stopped, and the spawn was killed at twelve minutes: a run that stopped inside the default budget read as a measurement. Pass the budget explicitly, keep the outer kill above it, resolve the entry each tool package declares instead of a platform shim - so a JavaScript entry runs on the Node binary already running the gate - and name the budget, the binary and the exit in every failure the spawn produces, because an exit code is not a mutant verdict on its own. A rejected run left no record. The gate threw before any report directory was created, the artifact step then reported an empty reports/mutation, and the reviewer was left a red mark and nothing to open. Write the rejection into that report root and into the job summary the platform provides, and downgrade a failed record to an advisory instead of letting it replace the rejection. The unwritable-summary case handed that write a directory, which no platform value of GITHUB_STEP_SUMMARY has ever been; it now hands a file path it cannot create and asserts the file it names, and a second case asserts that the file the platform does provide is appended to rather than replaced. The suite grows from 106 cases to 120 and all 120 pass. --- scripts/stryker-diff.mjs | 147 +++++++++-- scripts/stryker-diff.test.mjs | 482 +++++++++++++++++++++++++++++++++- 2 files changed, 610 insertions(+), 19 deletions(-) diff --git a/scripts/stryker-diff.mjs b/scripts/stryker-diff.mjs index 7dfe0006d2..65d6fb225d 100644 --- a/scripts/stryker-diff.mjs +++ b/scripts/stryker-diff.mjs @@ -13,6 +13,22 @@ export const MAX_MUTANTS = 400 export const PR_COMMENT_MARKER = "" export const PR_COMMENT_FILENAME = "pr-comment.md" +// A rejection is an outcome of the run, so it is recorded where the run records everything else. The +// artifact step uploads whatever the report root holds, and a rejection that leaves that directory +// empty leaves the reviewer a red mark and nothing to open. +export const REJECTION_FILENAME = "mutation-gate-rejection.md" +// The engine declares its own initial-test-run budget in its schema +// (node_modules/@stryker-mutator/core/schema/stryker-schema.json, key dryRunTimeoutMinutes) with a +// default of 5 minutes, and this repository's stryker.config.mjs does not override it. A whole-delta +// run that stops inside that default budget has not been measured: it has run out of budget. The tool +// therefore states the budget it passed instead of letting an engine default read as a verdict. +export const ENGINE_DEFAULT_DRY_RUN_MINUTES = 5 +export const DRY_RUN_TIMEOUT_MINUTES = 20 +// The outer kill stays above the budget handed to the engine, so the engine - not this tool - is the +// one that decides when an initial test run stops. +export const SPAWN_GRACE_MINUTES = 5 +export const spawnTimeoutMs = (DRY_RUN_TIMEOUT_MINUTES + SPAWN_GRACE_MINUTES) * 60 * 1_000 +const dryRunBudgetLabel = `initial-test-run budget ${DRY_RUN_TIMEOUT_MINUTES} min (engine schema default ${ENGINE_DEFAULT_DRY_RUN_MINUTES} min)` export const PACKAGE_CONFIGS = [ { @@ -351,9 +367,61 @@ export function shouldUseVitestRelated(packageEntry) { return (packageEntry.testFiles?.length ?? 0) === 0 && packageEntry.vitestRelated !== false } +// Read the executable entry the package itself declares in its package.json "bin" field, so the file +// that runs is the one the engine names rather than whichever extensionless shim libuv happens to +// resolve on this platform. Returns undefined when no such declaration exists, leaving the old shim +// lookup as the fallback. +export function resolveDeclaredBinEntry(repoRoot, runRoot, packageName, binName, packageRoot) { + for (const root of [ + ...new Set([runRoot, packageRoot, repoRoot].filter((candidate) => typeof candidate === "string")), + ]) { + const packageDir = path.join(root, "node_modules", packageName) + const manifestPath = path.join(packageDir, "package.json") + if (!fs.existsSync(manifestPath)) { + continue + } + let declared + try { + declared = JSON.parse(fs.readFileSync(manifestPath, "utf8")).bin + } catch { + continue + } + const entry = typeof declared === "string" ? declared : declared?.[binName] + if (typeof entry !== "string") { + continue + } + const target = path.join(packageDir, entry.replace(/^\.\//, "")) + if (fs.existsSync(target)) { + return target + } + } + return undefined +} + +// A bare node_modules/.bin/ file is the POSIX shell wrapper; on Windows the runnable files are +// the .cmd and .ps1 siblings, and picking between them is libuv's extension guess. When the target is +// already a JavaScript entry, run it with the Node binary that is executing this tool. +function spawnDeclaredEntry(binary, args, options) { + if (/\.[cm]?jsx?$/i.test(binary)) { + return spawnSync(process.execPath, [binary, ...args], options) + } + return spawnSync(binary, args, options) +} + +export function resolveStrykerBinary(repoRoot, runRoot) { + return ( + resolveDeclaredBinEntry(repoRoot, runRoot, "@stryker-mutator/core", "stryker") ?? + path.join(repoRoot, "node_modules/.bin/stryker") + ) +} + export function resolveVitestBinary(repoRoot, packageEntry) { const packageRoot = path.join(repoRoot, packageEntry.root) const runRoot = path.join(repoRoot, packageEntry.runRoot ?? packageEntry.root) + const declared = resolveDeclaredBinEntry(repoRoot, runRoot, "vitest", "vitest", packageRoot) + if (declared) { + return declared + } const candidates = [...new Set([runRoot, packageRoot, repoRoot])].map((root) => path.join(root, "node_modules/.bin/vitest"), ) @@ -370,7 +438,7 @@ export function discoverRelatedTestFiles(repoRoot, packageEntry, reportDirectory const outputFile = path.join(reportDirectory, "vitest-related.json") const configFile = path.relative(runRoot, path.join(packageRoot, packageEntry.vitestConfig)).replaceAll("\\", "/") const sourceFiles = [...new Set(packageEntry.selectors.map(selectorFile))] - const result = spawnSync( + const result = spawnDeclaredEntry( resolveVitestBinary(repoRoot, packageEntry), ["related", ...sourceFiles, "--run", "--config", configFile, "--reporter=json", `--outputFile=${outputFile}`], { @@ -417,11 +485,17 @@ function runStryker(repoRoot, packageEntry, reportRoot, dryRunOnly) { packageEntry.selectors.join(","), ] if (dryRunOnly) args.push("--dryRunOnly", "--reporters", "clear-text", "--logLevel", "info") + args.push("--dryRunTimeoutMinutes", String(DRY_RUN_TIMEOUT_MINUTES)) + + const strykerBinary = resolveStrykerBinary(repoRoot, runRoot) + console.log( + `${packageEntry.id}: Stryker ${dryRunOnly ? "preflight" : "run"} starting via ${path.relative(repoRoot, strykerBinary)} with ${dryRunBudgetLabel}; outer kill ${DRY_RUN_TIMEOUT_MINUTES + SPAWN_GRACE_MINUTES} min`, + ) - const result = spawnSync(path.join(repoRoot, "node_modules/.bin/stryker"), args, { + const result = spawnDeclaredEntry(strykerBinary, args, { cwd: runRoot, encoding: "utf8", - timeout: 12 * 60 * 1_000, + timeout: spawnTimeoutMs, maxBuffer: 50 * 1024 * 1024, env: { ...process.env, @@ -439,17 +513,17 @@ function runStryker(repoRoot, packageEntry, reportRoot, dryRunOnly) { const output = `${result.stdout ?? ""}${result.stderr ?? ""}` if (result.error?.code === "ETIMEDOUT") { throw new Error( - `${packageEntry.id} mutation run exceeded 12 minutes. Split the PR or obtain a maintainer-reviewed narrow exclusion.`, + `${packageEntry.id} mutation run exceeded the ${DRY_RUN_TIMEOUT_MINUTES + SPAWN_GRACE_MINUTES} minute outer kill while running under an ${dryRunBudgetLabel}. Split the PR or obtain a maintainer-reviewed narrow exclusion.`, ) } if (result.error) { throw new Error( - `${packageEntry.id} Stryker ${dryRunOnly ? "preflight" : "run"} could not start: ${result.error.message}`, + `${packageEntry.id} Stryker ${dryRunOnly ? "preflight" : "run"} could not start via ${path.relative(repoRoot, strykerBinary)} under an ${dryRunBudgetLabel}: ${result.error.message}`, ) } if (result.status !== 0) { throw new Error( - `${packageEntry.id} Stryker ${dryRunOnly ? "preflight" : "run"} failed:\n${stripAnsi(output).trim()}`, + `${packageEntry.id} Stryker ${dryRunOnly ? "preflight" : "run"} failed via ${path.relative(repoRoot, strykerBinary)} under an ${dryRunBudgetLabel} (exit ${result.status}, which is not a mutant verdict on its own):\n${stripAnsi(output).trim()}`, ) } @@ -707,6 +781,39 @@ export function appendSummary(rows, advisories, manifest, commentFile) { } } +export function formatRejection(baseSha, headSha, error) { + return [ + "## Changed-code mutation testing", + + "The gate rejected the run before it produced a mutation report.", + + `- Base: \`${baseSha.slice(0, 12)}\``, + `- Head: \`${headSha.slice(0, 12)}\``, + + `Mutation gate rejected: ${error.message}`, + ].join("\n") +} + +export function recordRejection(reportRoot, baseSha, headSha, error) { + const body = formatRejection(baseSha, headSha, error) + try { + fs.mkdirSync(reportRoot, { recursive: true }) + fs.writeFileSync(path.join(reportRoot, REJECTION_FILENAME), body) + } catch (error) { + console.warn( + `::warning title=Mutation test advisory::Could not write the mutation gate rejection file: ${escapeWorkflowData(error.message)}`, + ) + } + if (!process.env.GITHUB_STEP_SUMMARY) return + try { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, body) + } catch (error) { + console.warn( + `::warning title=Mutation test advisory::Could not write the job summary: ${escapeWorkflowData(error.message)}`, + ) + } +} + export function evaluateReport(report, packageEntry) { const counts = mutantCounts(report) const advisories = [] @@ -844,18 +951,24 @@ function main() { if (!baseSha || !headSha) throw new Error("--base and --head are required") const reportRoot = path.resolve(repoRoot, argument("--reports") ?? "reports/mutation") - const manifest = selectFromGit(repoRoot, baseSha, headSha) - if (manifest.packages.length === 0) { - appendSummary([], manifest.advisories, manifest, path.join(reportRoot, PR_COMMENT_FILENAME)) - console.log("No changed executable lines in mutation-tested packages; mutation testing is not applicable.") - return - } + try { + const manifest = selectFromGit(repoRoot, baseSha, headSha) + if (manifest.packages.length === 0) { + appendSummary([], manifest.advisories, manifest, path.join(reportRoot, PR_COMMENT_FILENAME)) + console.log("No changed executable lines in mutation-tested packages; mutation testing is not applicable.") + return + } - console.log( - `Mutation-testing ${manifest.packages.length} package(s) from merge base ${manifest.mergeBase.slice(0, 12)}: ` + - manifest.packages.map((entry) => `${entry.id} (${entry.changedExecutableLines} lines)`).join(", "), - ) - runManifest(repoRoot, manifest, reportRoot) + console.log( + `Mutation-testing ${manifest.packages.length} package(s) from merge base ${manifest.mergeBase.slice(0, 12)}: ` + + manifest.packages.map((entry) => `${entry.id} (${entry.changedExecutableLines} lines)`).join(", "), + ) + runManifest(repoRoot, manifest, reportRoot) + } catch (error) { + // The rejection writer never throws, so the attempt to record cannot replace the rejection. + recordRejection(reportRoot, baseSha, headSha, error) + throw error + } } if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { diff --git a/scripts/stryker-diff.test.mjs b/scripts/stryker-diff.test.mjs index 339e44e676..fc584de822 100644 --- a/scripts/stryker-diff.test.mjs +++ b/scripts/stryker-diff.test.mjs @@ -1,5 +1,5 @@ import assert from "node:assert/strict" -import { execFileSync } from "node:child_process" +import { execFileSync, spawnSync } from "node:child_process" import fs from "node:fs" import os from "node:os" import path from "node:path" @@ -9,11 +9,16 @@ import { runInNewContext } from "node:vm" import { parse as parseYaml } from "yaml" import { + DRY_RUN_TIMEOUT_MINUTES, + ENGINE_DEFAULT_DRY_RUN_MINUTES, MAX_CHANGED_LINES, MAX_MUTANTS, PACKAGE_CONFIGS, PR_COMMENT_FILENAME, PR_COMMENT_MARKER, + REJECTION_FILENAME, + SPAWN_GRACE_MINUTES, + spawnTimeoutMs, appendSummary, buildManifest, discoverRelatedTestFiles, @@ -30,6 +35,9 @@ import { parseNameStatus, parseVitestTestFiles, preferDirectTestFiles, + recordRejection, + resolveDeclaredBinEntry, + resolveStrykerBinary, resolveStrykerTempDir, resolveVitestBinary, shouldUseVitestRelated, @@ -868,14 +876,79 @@ describe("failure output", () => { it("does not fail when the GitHub job summary cannot be written", () => { const previousSummary = process.env.GITHUB_STEP_SUMMARY + const previousWarn = console.warn + const warnings = [] const summaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-summary-")) - process.env.GITHUB_STEP_SUMMARY = summaryDirectory + // The platform hands this write a regular file, so the simulation targets a file path it cannot + // create instead of a directory, which no platform value of GITHUB_STEP_SUMMARY has ever been. + const summaryTarget = path.join(summaryDirectory, "absent", "_summary.md") + process.env.GITHUB_STEP_SUMMARY = summaryTarget + console.warn = (warning) => warnings.push(warning) try { assert.doesNotThrow(() => appendSummary([], ["advisory"], {})) + // A file the platform promised and the gate could not create is the failure; the advisory names + // the file it failed on instead of staying quiet about it. + assert.deepEqual(warnings, [ + "::warning title=Mutation test advisory::advisory", + "::warning title=Mutation test advisory::Could not write the job summary: ENOENT: no such file or directory, open '" + + summaryTarget + + "'", + ]) + } finally { + if (previousSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY + else process.env.GITHUB_STEP_SUMMARY = previousSummary + console.warn = previousWarn + fs.rmSync(summaryDirectory, { recursive: true, force: true }) + } + }) + + it("names the file it could not append the job summary to", () => { + const previousSummary = process.env.GITHUB_STEP_SUMMARY + const previousWarn = console.warn + const warnings = [] + const summaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-summary-")) + const summaryTarget = path.join(summaryDirectory, "absent", "_summary.md") + process.env.GITHUB_STEP_SUMMARY = summaryTarget + console.warn = (warning) => warnings.push(warning) + + try { + appendSummary([], [], {}) + const summaryWarning = warnings.find((warning) => + warning.includes("::warning title=Mutation test advisory::Could not write the job summary"), + ) + assert.ok(summaryWarning, warnings.join(" | ")) + assert.ok(summaryWarning.includes(summaryTarget), summaryWarning) + assert.ok(summaryWarning.includes("ENOENT"), summaryWarning) + } finally { + if (previousSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY + else process.env.GITHUB_STEP_SUMMARY = previousSummary + console.warn = previousWarn + fs.rmSync(summaryDirectory, { recursive: true, force: true }) + } + }) + + it("appends the summary to the file the platform provides", () => { + const previousSummary = process.env.GITHUB_STEP_SUMMARY + const previousWarn = console.warn + const warnings = [] + const summaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-summary-")) + const summaryFile = path.join(summaryDirectory, "_summary.md") + fs.writeFileSync(summaryFile, "# written by an earlier step\n") + process.env.GITHUB_STEP_SUMMARY = summaryFile + console.warn = (warning) => warnings.push(warning) + const manifest = { baseSha: "a".repeat(40), headSha: "b".repeat(40) } + + try { + appendSummary([], ["manifest advisory"], manifest) + const written = fs.readFileSync(summaryFile, "utf8") + assert.ok(written.startsWith("# written by an earlier step\n"), written.slice(0, 120)) + assert.ok(written.includes(formatSummary([], ["manifest advisory"], manifest)), written.slice(0, 120)) + assert.deepEqual(warnings, ["::warning title=Mutation test advisory::manifest advisory"]) } finally { if (previousSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY else process.env.GITHUB_STEP_SUMMARY = previousSummary + console.warn = previousWarn fs.rmSync(summaryDirectory, { recursive: true, force: true }) } }) @@ -1624,3 +1697,408 @@ describe("mutation target resolution execution", () => { assert.equal(queries.length, 0) }) }) + +describe("gate rejection record", () => { + const rejectionMessage = + "extension has 2328 changed executable lines (limit 500). Split the PR or obtain a maintainer-reviewed narrow exclusion." + const rejectionError = () => new Error(rejectionMessage) + const headSha = "b".repeat(40) + const baseSha = "c".repeat(40) + + it("records the rejection in the report root the artifact step uploads", () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-rejection-")) + const reportRoot = path.join(repo, "reports", "mutation") + const previousSummary = process.env.GITHUB_STEP_SUMMARY + delete process.env.GITHUB_STEP_SUMMARY + + try { + assert.equal(fs.existsSync(reportRoot), false) + assert.doesNotThrow(() => recordRejection(reportRoot, baseSha, headSha, rejectionError())) + const body = fs.readFileSync(path.join(reportRoot, REJECTION_FILENAME), "utf8") + assert.ok(body.includes("Mutation gate rejected: " + rejectionMessage), body.slice(0, 200)) + assert.ok(body.includes("- Base: `" + baseSha.slice(0, 12) + "`"), body.slice(0, 200)) + assert.ok(body.includes("- Head: `" + headSha.slice(0, 12) + "`"), body.slice(0, 200)) + assert.deepEqual(fs.readdirSync(reportRoot), [REJECTION_FILENAME]) + } finally { + if (previousSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY + else process.env.GITHUB_STEP_SUMMARY = previousSummary + fs.rmSync(repo, { recursive: true, force: true }) + } + }) + + it("appends the rejection to the job summary the platform provides", () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-rejection-")) + const reportRoot = path.join(repo, "reports", "mutation") + const summaryFile = path.join(repo, "_summary.md") + const previousSummary = process.env.GITHUB_STEP_SUMMARY + const previousWarn = console.warn + const warnings = [] + fs.writeFileSync(summaryFile, "") + process.env.GITHUB_STEP_SUMMARY = summaryFile + console.warn = (warning) => warnings.push(warning) + + try { + recordRejection(reportRoot, baseSha, headSha, rejectionError()) + assert.deepEqual(warnings, []) + const written = fs.readFileSync(summaryFile, "utf8") + assert.ok(written.includes("Mutation gate rejected: " + rejectionMessage), written.slice(0, 200)) + } finally { + if (previousSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY + else process.env.GITHUB_STEP_SUMMARY = previousSummary + console.warn = previousWarn + fs.rmSync(repo, { recursive: true, force: true }) + } + }) + + it("leaves an advisory instead of throwing when the report root cannot be written", () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-rejection-")) + const occupied = path.join(repo, "reports") + const previousSummary = process.env.GITHUB_STEP_SUMMARY + const previousWarn = console.warn + const warnings = [] + fs.writeFileSync(occupied, "not a directory") + delete process.env.GITHUB_STEP_SUMMARY + console.warn = (warning) => warnings.push(warning) + + try { + assert.doesNotThrow(() => recordRejection(occupied, baseSha, headSha, rejectionError())) + assert.equal(warnings.length, 1, warnings.join(" | ")) + assert.ok( + warnings[0].startsWith( + "::warning title=Mutation test advisory::Could not write the mutation gate rejection file:", + ), + warnings[0], + ) + assert.ok(warnings[0].includes(occupied), warnings[0]) + } finally { + if (previousSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY + else process.env.GITHUB_STEP_SUMMARY = previousSummary + console.warn = previousWarn + fs.rmSync(repo, { recursive: true, force: true }) + } + }) + + it("leaves its record in the report root when the gate rejects a delta", () => { + // The script resolves its own imports from the repository it lives in, so the synthetic + // repository is built inside this one, where git ignores it and node can resolve typescript. + const scratchRoot = path.join(repositoryRoot, "reports", "mutation") + fs.mkdirSync(scratchRoot, { recursive: true }) + const scratch = fs.mkdtempSync(path.join(scratchRoot, "u917-rejection-")) + const run = (...args) => execFileSync("git", args, { cwd: scratch, encoding: "utf8" }).trim() + const write = (filePath, contents) => { + fs.mkdirSync(path.join(scratch, path.dirname(filePath)), { recursive: true }) + fs.writeFileSync(path.join(scratch, filePath), contents) + } + + try { + run("init", "--quiet", "--initial-branch", "main") + run("config", "user.email", "gate@example.com") + run("config", "user.name", "Gate") + run("config", "commit.gpgsign", "false") + run("config", "core.hooksPath", path.join(scratch, ".no-hooks")) + write("packages/core/src/value.ts", "export const value = () => 1\n") + run("add", ".") + run("commit", "--quiet", "-m", "initial") + const beforeSha = run("rev-parse", "HEAD") + + const lines = Array.from( + { length: MAX_CHANGED_LINES + 50 }, + (_, index) => "export const value" + index + " = " + index, + ) + write("packages/core/src/value.ts", lines.join("\n") + "\n") + run("add", ".") + run("commit", "--quiet", "-m", "delta above the changed-line limit") + const afterSha = run("rev-parse", "HEAD") + + fs.mkdirSync(path.join(scratch, "scripts"), { recursive: true }) + fs.copyFileSync( + path.join(repositoryRoot, "scripts/stryker-diff.mjs"), + path.join(scratch, "scripts/stryker-diff.mjs"), + ) + const result = spawnSync( + process.execPath, + ["scripts/stryker-diff.mjs", "ci", "--base", beforeSha, "--head", afterSha, "--reports", "rejection"], + { cwd: scratch, encoding: "utf8", timeout: 120000 }, + ) + + assert.equal(result.status, 1, String(result.stderr).slice(0, 300)) + assert.ok( + String(result.stderr).includes("changed executable lines (limit " + MAX_CHANGED_LINES + ")"), + String(result.stderr).slice(0, 300), + ) + const reportDirectory = path.join(scratch, "rejection") + const recorded = fs.readFileSync(path.join(reportDirectory, REJECTION_FILENAME), "utf8") + assert.ok(recorded.includes("Mutation gate rejected: core has "), recorded.slice(0, 240)) + assert.deepEqual(fs.readdirSync(reportDirectory), [REJECTION_FILENAME]) + } finally { + fs.rmSync(scratch, { recursive: true, force: true }) + } + }) +}) + +describe("declared binary resolution", () => { + const writeDeclaredPackage = (repo, packageName, bin, contents = "") => { + const packageDir = path.join(repo, "node_modules", packageName) + fs.mkdirSync(packageDir, { recursive: true }) + fs.writeFileSync(path.join(packageDir, "package.json"), JSON.stringify({ name: packageName, bin })) + const entry = typeof bin === "string" ? bin : Object.values(bin)[0] + const entryPath = path.join(packageDir, entry.replace(/^\.\//, "")) + fs.mkdirSync(path.dirname(entryPath), { recursive: true }) + fs.writeFileSync(entryPath, contents) + return entryPath + } + + it("reads the executable entry the package declares in both npm forms", () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-declared-")) + + try { + const mapped = writeDeclaredPackage(repo, "vitest", { vitest: "./vitest.mjs" }) + assert.equal(resolveDeclaredBinEntry(repo, repo, "vitest", "vitest", repo), mapped) + const single = writeDeclaredPackage(repo, "@stryker-mutator/core", "./bin/stryker.js") + assert.equal(resolveDeclaredBinEntry(repo, repo, "@stryker-mutator/core", "stryker", repo), single) + } finally { + fs.rmSync(repo, { recursive: true, force: true }) + } + }) + + it("gives up when the declared file or the declaration is missing", () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-declared-")) + const packageDir = path.join(repo, "node_modules", "vitest") + const packageJson = path.join(packageDir, "package.json") + + try { + const entry = writeDeclaredPackage(repo, "vitest", { vitest: "./vitest.mjs" }) + fs.rmSync(entry) + assert.equal(resolveDeclaredBinEntry(repo, repo, "vitest", "vitest", repo), undefined) + fs.writeFileSync(packageJson, JSON.stringify({ name: "vitest" })) + assert.equal(resolveDeclaredBinEntry(repo, repo, "vitest", "vitest", repo), undefined) + fs.writeFileSync(packageJson, "{ not json") + assert.equal(resolveDeclaredBinEntry(repo, repo, "vitest", "vitest", repo), undefined) + } finally { + fs.rmSync(repo, { recursive: true, force: true }) + } + }) + + it("prefers the declared entry over the platform shim", () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-declared-")) + const strykerShim = path.join(repo, "node_modules", ".bin", "stryker") + fs.mkdirSync(path.dirname(strykerShim), { recursive: true }) + fs.writeFileSync(strykerShim, "") + const strykerEntry = writeDeclaredPackage(repo, "@stryker-mutator/core", { stryker: "./bin/stryker.js" }) + const vitestEntry = writeDeclaredPackage(repo, "vitest", { vitest: "./vitest.mjs" }) + const packageShim = path.join(repo, "packages", "core", "node_modules", ".bin", "vitest") + fs.mkdirSync(path.dirname(packageShim), { recursive: true }) + fs.writeFileSync(packageShim, "") + const packageEntry = { + id: "core", + root: "packages/core", + vitestConfig: "vitest.unit.config.ts", + } + + try { + assert.equal(resolveStrykerBinary(repo, repo), strykerEntry) + assert.equal(resolveVitestBinary(repo, packageEntry), vitestEntry) + } finally { + fs.rmSync(repo, { recursive: true, force: true }) + } + }) + + it("falls back to the repository shim when nothing is declared", () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-declared-")) + const strykerShim = path.join(repo, "node_modules", ".bin", "stryker") + fs.mkdirSync(path.dirname(strykerShim), { recursive: true }) + fs.writeFileSync(strykerShim, "") + + try { + assert.equal(resolveStrykerBinary(repo, repo), strykerShim) + assert.equal( + resolveVitestBinary(repo, { id: "core", root: "packages/core", vitestConfig: "vitest.unit.config.ts" }), + path.join(repo, "node_modules", ".bin", "vitest"), + ) + } finally { + fs.rmSync(repo, { recursive: true, force: true }) + } + }) + + it("runs a JavaScript entry with the Node binary that is running the gate", () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-vitest-entry-")) + const reportDirectory = path.join(repo, "reports") + const previousTestFile = process.env.U917_VITEST_TEST_FILE + const fake = [ + 'import fs from "node:fs"', + "const args = process.argv.slice(2)", + 'const outputFile = args.find((arg) => arg.startsWith("--outputFile=")).slice("--outputFile=".length)', + 'fs.writeFileSync(outputFile + ".exec.json", JSON.stringify({ execPath: process.execPath, argv: process.argv.slice(1) }))', + "fs.writeFileSync(outputFile, JSON.stringify({ testResults: [{ name: process.env.U917_VITEST_TEST_FILE }] }))", + ].join("\n") + const entry = writeDeclaredPackage(repo, "vitest", { vitest: "./vitest.mjs" }, fake) + fs.mkdirSync(path.join(repo, "src", "utils"), { recursive: true }) + fs.mkdirSync(reportDirectory, { recursive: true }) + const packageEntry = { + id: "extension", + root: "src", + vitestConfig: "vitest.config.ts", + selectors: ["utils/value.ts:1-1"], + } + process.env.U917_VITEST_TEST_FILE = path.join(repo, "src", "utils", "value.test.ts") + + try { + assert.deepEqual(discoverRelatedTestFiles(repo, packageEntry, reportDirectory), ["utils/value.test.ts"]) + const record = JSON.parse( + fs.readFileSync(path.join(reportDirectory, "vitest-related.json.exec.json"), "utf8"), + ) + assert.equal(record.execPath, process.execPath) + assert.equal(record.argv[0], entry) + assert.equal(record.argv[1], "related") + assert.ok( + record.argv.some((arg) => arg.startsWith("--outputFile=")), + record.argv.join(" "), + ) + } finally { + if (previousTestFile === undefined) delete process.env.U917_VITEST_TEST_FILE + else process.env.U917_VITEST_TEST_FILE = previousTestFile + fs.rmSync(repo, { recursive: true, force: true }) + } + }) +}) + +describe("initial-test-run budget", () => { + const packageEntry = { + id: "core", + root: "packages/core", + vitestConfig: "vitest.unit.config.ts", + selectors: ["src/value.ts:1-1"], + changedExecutableLines: 1, + } + const writeStrykerStub = (repo, exitCode) => { + const packageDir = path.join(repo, "node_modules", "@stryker-mutator", "core") + fs.mkdirSync(packageDir, { recursive: true }) + fs.writeFileSync( + path.join(packageDir, "package.json"), + JSON.stringify({ name: "@stryker-mutator/core", bin: { stryker: "./bin/stryker.js" } }), + ) + const entryPath = path.join(packageDir, "bin", "stryker.js") + fs.mkdirSync(path.dirname(entryPath), { recursive: true }) + const stub = [ + 'const fs = require("node:fs")', + "const args = process.argv.slice(2)", + 'const kind = args.includes("--dryRunOnly") ? "preflight" : "run"', + 'fs.writeFileSync(process.env.U917_STRYKER_RECORD + "." + kind + ".json", JSON.stringify({ execPath: process.execPath, argv1: process.argv[1], args }))', + 'if (kind === "preflight") console.log("Instrumented 1 source file(s) with 1 mutant(s)")', + "process.exit(" + exitCode + ")", + ].join("\n") + fs.writeFileSync(entryPath, stub) + return entryPath + } + const readInvocation = (recordPath, kind) => JSON.parse(fs.readFileSync(recordPath + "." + kind + ".json", "utf8")) + + it("keeps the outer kill above the budget handed to the engine", () => { + assert.equal(DRY_RUN_TIMEOUT_MINUTES, 20) + assert.equal(ENGINE_DEFAULT_DRY_RUN_MINUTES, 5) + assert.equal(SPAWN_GRACE_MINUTES, 5) + assert.equal(spawnTimeoutMs, (DRY_RUN_TIMEOUT_MINUTES + SPAWN_GRACE_MINUTES) * 60 * 1000) + assert.ok(spawnTimeoutMs > DRY_RUN_TIMEOUT_MINUTES * 60 * 1000) + }) + + it("hands every Stryker invocation the budget it names and logs the binary it used", () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-budget-")) + const reportRoot = path.join(repo, "reports") + const recordPath = path.join(repo, "argv.json") + const previousRecord = process.env.U917_STRYKER_RECORD + const previousLog = console.log + const previousWarn = console.warn + const logs = [] + process.env.U917_STRYKER_RECORD = recordPath + fs.mkdirSync(path.join(repo, "packages", "core", "src"), { recursive: true }) + const entry = writeStrykerStub(repo, 0) + console.log = (line) => logs.push(String(line)) + console.warn = () => {} + + try { + const rows = runManifest(repo, { packages: [{ ...packageEntry }] }, reportRoot, { + readMutationReport: () => ({ files: {} }), + }) + assert.equal(rows.length, 1) + const preflight = readInvocation(recordPath, "preflight") + const run = readInvocation(recordPath, "run") + for (const invocation of [preflight, run]) { + const budgetAt = invocation.args.indexOf("--dryRunTimeoutMinutes") + assert.ok(budgetAt >= 0, invocation.args.join(" ")) + assert.equal(invocation.args[budgetAt + 1], String(DRY_RUN_TIMEOUT_MINUTES)) + assert.equal(invocation.execPath, process.execPath) + assert.equal(invocation.argv1, entry) + } + assert.equal(preflight.args.includes("--dryRunOnly"), true) + assert.equal(run.args.includes("--dryRunOnly"), false) + const reported = logs.join("\n") + assert.ok( + logs.some( + (line) => + line.includes("Stryker preflight starting via") && line.includes(path.relative(repo, entry)), + ), + reported.slice(0, 400), + ) + assert.ok( + logs.some((line) => + line.includes( + "initial-test-run budget " + + DRY_RUN_TIMEOUT_MINUTES + + " min (engine schema default " + + ENGINE_DEFAULT_DRY_RUN_MINUTES + + " min)", + ), + ), + reported.slice(0, 400), + ) + } finally { + if (previousRecord === undefined) delete process.env.U917_STRYKER_RECORD + else process.env.U917_STRYKER_RECORD = previousRecord + console.log = previousLog + console.warn = previousWarn + fs.rmSync(repo, { recursive: true, force: true }) + } + }) + + it("names the budget, the binary, and that the exit is not a verdict when the engine fails", () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-budget-")) + const reportRoot = path.join(repo, "reports") + const recordPath = path.join(repo, "argv.json") + const previousRecord = process.env.U917_STRYKER_RECORD + const previousLog = console.log + const previousWarn = console.warn + const warnings = [] + process.env.U917_STRYKER_RECORD = recordPath + fs.mkdirSync(path.join(repo, "packages", "core", "src"), { recursive: true }) + const entry = writeStrykerStub(repo, 1) + console.log = () => {} + console.warn = (warning) => warnings.push(String(warning)) + + try { + const rows = runManifest(repo, { packages: [{ ...packageEntry }] }, reportRoot, { + readMutationReport: () => ({ files: {} }), + }) + assert.equal(rows[0].result, "Advisory incomplete") + const failure = warnings.find((warning) => warning.includes("Stryker preflight failed via")) + assert.ok(failure, warnings.join(" | ").slice(0, 400)) + assert.ok( + failure.includes( + "initial-test-run budget " + + DRY_RUN_TIMEOUT_MINUTES + + " min (engine schema default " + + ENGINE_DEFAULT_DRY_RUN_MINUTES + + " min)", + ), + failure.slice(0, 400), + ) + assert.ok(failure.includes("(exit 1, which is not a mutant verdict on its own)"), failure.slice(0, 400)) + assert.ok(failure.includes(path.relative(repo, entry)), failure.slice(0, 400)) + assert.equal(fs.existsSync(recordPath + ".run.json"), false) + } finally { + if (previousRecord === undefined) delete process.env.U917_STRYKER_RECORD + else process.env.U917_STRYKER_RECORD = previousRecord + console.log = previousLog + console.warn = previousWarn + fs.rmSync(repo, { recursive: true, force: true }) + } + }) +}) From f41968f973e764ff089ed4022c0f841a48199b25 Mon Sep 17 00:00:00 2001 From: "user.email" <136036952+easonLiangWorldedtech@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:29:23 +0800 Subject: [PATCH 2/3] scripts: give the mutation gate a manifest budget and its suite a summary of its own The gate handed every Stryker invocation the same 20-minute initial-test-run budget and kept no clock of its own, while the mutation-diff job carries timeout-minutes: 30. Measured against the previous head, a two-package manifest made four invocations, every one handed 20 minutes with a 25-minute outer kill, which is past the job before any package is found over either ceiling, and a job cancelled at its limit writes no rejection record and appends no job summary. The manifest now measures the time it has spent, hands each further run only the budget that is left, keeps the grace minutes as the headroom between that budget and the outer kill, and holds back three minutes for the rejection record and the job summary. Measured on the same footing it made three invocations at 20, 13 and 4 minutes with outer kills of 25, 18 and 9, and refused the fourth launch: that refusal names the spent budget, the job limit and the reserve, goes through the existing rejection record and job summary as the run's own outcome, and is not reported as an incomplete package. The suite also ran the gate inside the very job that publishes a summary, since the mutation-diff job runs this file as its own step. Running the previous test file with a job summary in place left 7,553 bytes in it, including the words "Mutation gate rejected", written from synthetic test data while the step passed. The suite now runs against a summary file of its own and the last describe reads the inherited one back: measured 0 bytes left in it, with each gate test given its own file as well. node --test scripts/stryker-diff.test.mjs: 125 tests, 26 suites, 125 pass, 0 fail (runner line 5136.768 ms) where the head before these two fixes printed 120 tests, 24 suites, 120 pass, 0 fail. Prettier 3.9.10 --check reports both paths clean, node --check parses both, src/eslint-suppressions.json stays at blob d5b845997a775924f598d6b47793fe226b783c61, and no changeset or changelog entry is added. --- scripts/stryker-diff.mjs | 62 ++++++-- scripts/stryker-diff.test.mjs | 277 +++++++++++++++++++++++++++++++++- 2 files changed, 328 insertions(+), 11 deletions(-) diff --git a/scripts/stryker-diff.mjs b/scripts/stryker-diff.mjs index 65d6fb225d..5e61f6a51d 100644 --- a/scripts/stryker-diff.mjs +++ b/scripts/stryker-diff.mjs @@ -28,7 +28,37 @@ export const DRY_RUN_TIMEOUT_MINUTES = 20 // one that decides when an initial test run stops. export const SPAWN_GRACE_MINUTES = 5 export const spawnTimeoutMs = (DRY_RUN_TIMEOUT_MINUTES + SPAWN_GRACE_MINUTES) * 60 * 1_000 -const dryRunBudgetLabel = `initial-test-run budget ${DRY_RUN_TIMEOUT_MINUTES} min (engine schema default ${ENGINE_DEFAULT_DRY_RUN_MINUTES} min)` +// The mutation-diff job in .github/workflows/mutation-testing.yml carries timeout-minutes: 30, and a job +// cancelled at that limit stops the script before it can write the rejection record or append the job +// summary. Two sequential initial test runs of DRY_RUN_TIMEOUT_MINUTES each are already over that +// limit, so the manifest measures the time it has spent itself and hands each further run only the +// budget that is left: SPAWN_GRACE_MINUTES stays the headroom between the budget handed to the engine +// and the outer kill, and TERMINUS_RESERVE_MINUTES is held back for the rejection record and the +// job summary. A run is therefore never started unless it can still finish before the job is cancelled. +export const JOB_TIMEOUT_MINUTES = 30 +export const TERMINUS_RESERVE_MINUTES = 3 +export const MANIFEST_BUDGET_MINUTES = JOB_TIMEOUT_MINUTES - TERMINUS_RESERVE_MINUTES +export const manifestBudgetMs = MANIFEST_BUDGET_MINUTES * 60 * 1_000 + +export function manifestRunBudgetMinutes(spentMinutes, budgetMinutes = MANIFEST_BUDGET_MINUTES) { + const spareMinutes = budgetMinutes - spentMinutes + if (spareMinutes <= SPAWN_GRACE_MINUTES) return 0 + return Math.min(DRY_RUN_TIMEOUT_MINUTES, Math.floor(spareMinutes - SPAWN_GRACE_MINUTES)) +} + +export function manifestBudgetError(packageEntry, phase, spentMinutes, budgetMinutes) { + const error = new Error( + `${packageEntry.id} ${phase} was not launched: the manifest had spent ${Math.floor(spentMinutes)} min of its ${budgetMinutes} min budget, the mutation-diff job carries ${JOB_TIMEOUT_MINUTES} min and ${TERMINUS_RESERVE_MINUTES} min is held back for the rejection record and the job summary. ` + + "Split the PR or obtain a maintainer-reviewed narrow exclusion.", + ) + // The per-package catch reports a failed package as an advisory row; a manifest that ran out of its + // budget is not a failed package, it is the run itself ending, so the flag keeps it on the way out. + error.manifestBudgetStop = true + return error +} + +const dryRunBudgetLabel = (runMinutes) => + `initial-test-run budget ${runMinutes} min (engine schema default ${ENGINE_DEFAULT_DRY_RUN_MINUTES} min)` export const PACKAGE_CONFIGS = [ { @@ -471,7 +501,7 @@ export function discoverRelatedTestFiles(repoRoot, packageEntry, reportDirectory return testFiles } -function runStryker(repoRoot, packageEntry, reportRoot, dryRunOnly) { +function runStryker(repoRoot, packageEntry, reportRoot, dryRunOnly, runMinutes = DRY_RUN_TIMEOUT_MINUTES) { const packageRoot = path.join(repoRoot, packageEntry.root) const runRoot = path.join(repoRoot, packageEntry.runRoot ?? packageEntry.root) const reportDirectory = path.join(reportRoot, packageEntry.id) @@ -485,17 +515,17 @@ function runStryker(repoRoot, packageEntry, reportRoot, dryRunOnly) { packageEntry.selectors.join(","), ] if (dryRunOnly) args.push("--dryRunOnly", "--reporters", "clear-text", "--logLevel", "info") - args.push("--dryRunTimeoutMinutes", String(DRY_RUN_TIMEOUT_MINUTES)) + args.push("--dryRunTimeoutMinutes", String(runMinutes)) const strykerBinary = resolveStrykerBinary(repoRoot, runRoot) console.log( - `${packageEntry.id}: Stryker ${dryRunOnly ? "preflight" : "run"} starting via ${path.relative(repoRoot, strykerBinary)} with ${dryRunBudgetLabel}; outer kill ${DRY_RUN_TIMEOUT_MINUTES + SPAWN_GRACE_MINUTES} min`, + `${packageEntry.id}: Stryker ${dryRunOnly ? "preflight" : "run"} starting via ${path.relative(repoRoot, strykerBinary)} with ${dryRunBudgetLabel(runMinutes)}; outer kill ${runMinutes + SPAWN_GRACE_MINUTES} min`, ) const result = spawnDeclaredEntry(strykerBinary, args, { cwd: runRoot, encoding: "utf8", - timeout: spawnTimeoutMs, + timeout: (runMinutes + SPAWN_GRACE_MINUTES) * 60 * 1_000, maxBuffer: 50 * 1024 * 1024, env: { ...process.env, @@ -513,17 +543,17 @@ function runStryker(repoRoot, packageEntry, reportRoot, dryRunOnly) { const output = `${result.stdout ?? ""}${result.stderr ?? ""}` if (result.error?.code === "ETIMEDOUT") { throw new Error( - `${packageEntry.id} mutation run exceeded the ${DRY_RUN_TIMEOUT_MINUTES + SPAWN_GRACE_MINUTES} minute outer kill while running under an ${dryRunBudgetLabel}. Split the PR or obtain a maintainer-reviewed narrow exclusion.`, + `${packageEntry.id} mutation run exceeded the ${runMinutes + SPAWN_GRACE_MINUTES} minute outer kill while running under an ${dryRunBudgetLabel(runMinutes)}. Split the PR or obtain a maintainer-reviewed narrow exclusion.`, ) } if (result.error) { throw new Error( - `${packageEntry.id} Stryker ${dryRunOnly ? "preflight" : "run"} could not start via ${path.relative(repoRoot, strykerBinary)} under an ${dryRunBudgetLabel}: ${result.error.message}`, + `${packageEntry.id} Stryker ${dryRunOnly ? "preflight" : "run"} could not start via ${path.relative(repoRoot, strykerBinary)} under an ${dryRunBudgetLabel(runMinutes)}: ${result.error.message}`, ) } if (result.status !== 0) { throw new Error( - `${packageEntry.id} Stryker ${dryRunOnly ? "preflight" : "run"} failed via ${path.relative(repoRoot, strykerBinary)} under an ${dryRunBudgetLabel} (exit ${result.status}, which is not a mutant verdict on its own):\n${stripAnsi(output).trim()}`, + `${packageEntry.id} Stryker ${dryRunOnly ? "preflight" : "run"} failed via ${path.relative(repoRoot, strykerBinary)} under an ${dryRunBudgetLabel(runMinutes)} (exit ${result.status}, which is not a mutant verdict on its own):\n${stripAnsi(output).trim()}`, ) } @@ -839,11 +869,20 @@ export function runManifest( { runMutation = runStryker, readMutationReport = (reportPath) => JSON.parse(fs.readFileSync(reportPath, "utf8")), + now = () => Date.now(), + budgetMinutes = MANIFEST_BUDGET_MINUTES, } = {}, ) { const rows = [] const advisories = [...(manifest.advisories ?? [])] const annotationState = { total: 0, perFile: new Map() } + const manifestStartMs = now() + const launchBudgetMinutes = (packageEntry, phase) => { + const spentMinutes = (now() - manifestStartMs) / 60_000 + const minutes = manifestRunBudgetMinutes(spentMinutes, budgetMinutes) + if (minutes === 0) throw manifestBudgetError(packageEntry, phase, spentMinutes, budgetMinutes) + return minutes + } for (const packageEntry of manifest.packages) { let counts @@ -856,7 +895,9 @@ export function runManifest( if (packageEntry.discoverRelatedTests) { packageEntry.testFiles = discoverRelatedTestFiles(repoRoot, packageEntry, reportDirectory) } - const preflightOutput = stripAnsi(runMutation(repoRoot, packageEntry, reportRoot, true)) + const preflightOutput = stripAnsi( + runMutation(repoRoot, packageEntry, reportRoot, true, launchBudgetMinutes(packageEntry, "preflight")), + ) const mutantMatch = /Instrumented \d+ source file\(s\) with (\d+) mutant\(s\)/.exec(preflightOutput) if (!mutantMatch) throw new Error(`${packageEntry.id} preflight did not report a mutant count`) const generatedMutants = Number(mutantMatch[1]) @@ -886,7 +927,7 @@ export function runManifest( continue } - runMutation(repoRoot, packageEntry, reportRoot, false) + runMutation(repoRoot, packageEntry, reportRoot, false, launchBudgetMinutes(packageEntry, "mutation run")) const jsonReportPath = path.join(reportRoot, packageEntry.id, "mutation.json") const report = readMutationReport(jsonReportPath) packageEntry.testFiles = testsFromMutationReport(report, packageEntry.testFiles) @@ -911,6 +952,7 @@ export function runManifest( result: counts.blocking.length > 0 || counts.advisories.length > 0 ? "Advisory findings" : "Passed", }) } catch (error) { + if (error?.manifestBudgetStop === true) throw error advisories.push(error.message) rows.push({ id: packageEntry.id, diff --git a/scripts/stryker-diff.test.mjs b/scripts/stryker-diff.test.mjs index fc584de822..65ae30b82f 100644 --- a/scripts/stryker-diff.test.mjs +++ b/scripts/stryker-diff.test.mjs @@ -17,7 +17,13 @@ import { PR_COMMENT_FILENAME, PR_COMMENT_MARKER, REJECTION_FILENAME, + JOB_TIMEOUT_MINUTES, + MANIFEST_BUDGET_MINUTES, SPAWN_GRACE_MINUTES, + TERMINUS_RESERVE_MINUTES, + manifestBudgetError, + manifestBudgetMs, + manifestRunBudgetMinutes, spawnTimeoutMs, appendSummary, buildManifest, @@ -50,6 +56,18 @@ import { const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..") +// The gate publishes whatever it appends to GITHUB_STEP_SUMMARY, and this suite runs the gate against +// synthetic deltas from inside the very job that publishes a summary: the mutation-diff job runs this +// file as its own step, so the step already owns a job summary by the time the suite starts. Without a +// summary of its own, a synthetic table or a synthetic rejection from test data is published as if it +// were that runs own outcome. The suite points the variable at a file of its own, and the last describe +// proves the file the suite inherited stayed empty. +const suiteSummaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-suite-summary-")) +const suiteSummary = path.join(suiteSummaryRoot, "suite-summary.md") +fs.writeFileSync(suiteSummary, "") +const inheritedJobSummary = process.env.GITHUB_STEP_SUMMARY +process.env.GITHUB_STEP_SUMMARY = suiteSummary +process.once("exit", () => fs.rmSync(suiteSummaryRoot, { recursive: true, force: true })) describe("mutation testing workflow", () => { const readWorkflow = () => fs.readFileSync(path.join(repositoryRoot, ".github/workflows/mutation-testing.yml"), "utf8") @@ -1790,6 +1808,18 @@ describe("gate rejection record", () => { fs.writeFileSync(path.join(scratch, filePath), contents) } + // The gate appends its rejection to GITHUB_STEP_SUMMARY, and this test runs the gate in a child + // process that would otherwise inherit the summary of the step running this suite. A rejection + // produced from synthetic test data would then be published as if it were the runs own outcome + // while the job passed. The child is pointed at a summary of its own and the file it would have + // inherited is kept as a probe that has to stay empty. The declarations sit outside the try + // because the finally block has to reach them. + const isolatedSummary = path.join(scratch, "isolated-step-summary.md") + const inheritedSummary = path.join(scratch, "inherited-step-summary.md") + fs.writeFileSync(isolatedSummary, "") + fs.writeFileSync(inheritedSummary, "") + const previousSummary = process.env.GITHUB_STEP_SUMMARY + process.env.GITHUB_STEP_SUMMARY = inheritedSummary try { run("init", "--quiet", "--initial-branch", "main") run("config", "user.email", "gate@example.com") @@ -1818,7 +1848,12 @@ describe("gate rejection record", () => { const result = spawnSync( process.execPath, ["scripts/stryker-diff.mjs", "ci", "--base", beforeSha, "--head", afterSha, "--reports", "rejection"], - { cwd: scratch, encoding: "utf8", timeout: 120000 }, + { + cwd: scratch, + encoding: "utf8", + timeout: 120000, + env: { ...process.env, GITHUB_STEP_SUMMARY: isolatedSummary }, + }, ) assert.equal(result.status, 1, String(result.stderr).slice(0, 300)) @@ -1830,7 +1865,16 @@ describe("gate rejection record", () => { const recorded = fs.readFileSync(path.join(reportDirectory, REJECTION_FILENAME), "utf8") assert.ok(recorded.includes("Mutation gate rejected: core has "), recorded.slice(0, 240)) assert.deepEqual(fs.readdirSync(reportDirectory), [REJECTION_FILENAME]) + assert.equal( + fs.readFileSync(inheritedSummary, "utf8"), + "", + "the gate child appended to the job summary it inherited instead of the one it was given", + ) + const published = fs.readFileSync(isolatedSummary, "utf8") + assert.ok(published.includes("Mutation gate rejected: core has "), published.slice(0, 240)) } finally { + if (previousSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY + else process.env.GITHUB_STEP_SUMMARY = previousSummary fs.rmSync(scratch, { recursive: true, force: true }) } }) @@ -2102,3 +2146,234 @@ describe("initial-test-run budget", () => { } }) }) + +describe("manifest wall-clock budget", () => { + const packageEntry = (id, root) => ({ + id, + root, + vitestConfig: "vitest.unit.config.ts", + selectors: ["src/value.ts:1-1"], + changedExecutableLines: 1, + }) + const preflightLine = "Instrumented 1 source file(s) with 1 mutant(s)" + + it("holds the manifest budget to the job timeout the workflow prints", () => { + const workflow = fs.readFileSync(path.join(repositoryRoot, ".github/workflows/mutation-testing.yml"), "utf8") + const printed = Number((/timeout-minutes: *(\d+)/.exec(workflow) || [])[1]) + assert.ok(Number.isFinite(printed), "the mutation-diff job must print a timeout-minutes value") + assert.equal(JOB_TIMEOUT_MINUTES, printed, "the constant has to name the job limit it is measured against") + assert.equal(TERMINUS_RESERVE_MINUTES, 3) + assert.equal(MANIFEST_BUDGET_MINUTES, JOB_TIMEOUT_MINUTES - TERMINUS_RESERVE_MINUTES) + assert.equal(manifestBudgetMs, MANIFEST_BUDGET_MINUTES * 60 * 1_000) + // Two sequential initial test runs, each holding the full budget plus its grace, are already past + // the job limit, which is what the manifest budget exists for. + assert.ok( + 2 * (DRY_RUN_TIMEOUT_MINUTES + SPAWN_GRACE_MINUTES) > JOB_TIMEOUT_MINUTES, + "two full runs have to be too much for the job or the manifest budget would be pointless", + ) + assert.ok( + MANIFEST_BUDGET_MINUTES + TERMINUS_RESERVE_MINUTES <= JOB_TIMEOUT_MINUTES, + "the manifest budget plus the reserve has to stay inside the job", + ) + }) + + it("hands a run only the budget the manifest has left and reports none left as a stop", () => { + assert.equal(manifestRunBudgetMinutes(0), DRY_RUN_TIMEOUT_MINUTES) + assert.equal(manifestRunBudgetMinutes(13.5), 8) + assert.equal(manifestRunBudgetMinutes(18), 4) + assert.equal(manifestRunBudgetMinutes(21.9), 0) + assert.equal(manifestRunBudgetMinutes(27), 0) + // The outer kill must stay above the budget handed over, or this tool and not the engine would be + // the one that stops an initial test run. + for (const spent of [0, 13.5, 18]) { + const minutes = manifestRunBudgetMinutes(spent) + const roomMinutes = MANIFEST_BUDGET_MINUTES - spent + assert.ok( + (minutes + SPAWN_GRACE_MINUTES) * 60_000 <= roomMinutes * 60_000, + "a run launched with " + spent + " min spent has to be able to finish inside the job", + ) + } + const stop = manifestBudgetError( + packageEntry("core", "packages/core"), + "mutation run", + 27, + MANIFEST_BUDGET_MINUTES, + ) + assert.equal(stop.manifestBudgetStop, true) + assert.ok(stop.message.includes("was not launched"), stop.message) + assert.ok(stop.message.includes("27 min budget"), stop.message) + assert.ok(stop.message.includes("mutation-diff job carries 30 min"), stop.message) + assert.ok( + stop.message.includes("3 min is held back for the rejection record and the job summary"), + stop.message, + ) + }) + + it("stops launching rather than walk past the job, and records the stop as a rejection", () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-manifest-budget-")) + const reportRoot = path.join(repo, "reports") + const launches = [] + const clockValues = [0, 0, 9 * 60_000, 18 * 60_000, 27 * 60_000] + let tick = 0 + const previousSummary = process.env.GITHUB_STEP_SUMMARY + const previousWarn = console.warn + const warnings = [] + console.warn = (warning) => warnings.push(String(warning)) + delete process.env.GITHUB_STEP_SUMMARY + let failure + try { + runManifest( + repo, + { packages: [packageEntry("core", "packages/core"), packageEntry("webview", "packages/webview")] }, + reportRoot, + { + runMutation: (_repoRoot, entry, _reportRoot, dryRunOnly, runMinutes) => { + launches.push([entry.id, dryRunOnly, runMinutes]) + return dryRunOnly ? preflightLine : "" + }, + readMutationReport: () => ({ files: {} }), + now: () => { + const value = clockValues[tick] + assert.ok(tick < clockValues.length, "the injected clock ran out of readings") + tick += 1 + return value + }, + }, + ) + } catch (error) { + failure = error + } + try { + assert.ok(failure, "the manifest should have refused to launch a fourth run") + assert.equal(failure.manifestBudgetStop, true) + assert.deepEqual( + launches, + [ + ["core", true, DRY_RUN_TIMEOUT_MINUTES], + ["core", false, 13], + ["webview", true, 4], + ], + JSON.stringify(launches), + ) + assert.equal( + launches.some((launch) => launch[0] === "webview" && launch[1] === false), + false, + "the second package started its mutation run with no manifest budget left", + ) + const rejectionRoot = path.join(repo, "rejection") + recordRejection(rejectionRoot, "b".repeat(40), "c".repeat(40), failure) + const recorded = fs.readFileSync(path.join(rejectionRoot, REJECTION_FILENAME), "utf8") + assert.ok(recorded.includes("was not launched"), recorded.slice(0, 240)) + assert.equal(warnings.length, 0, warnings.join(" | ")) + } finally { + if (previousSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY + else process.env.GITHUB_STEP_SUMMARY = previousSummary + console.warn = previousWarn + fs.rmSync(repo, { recursive: true, force: true }) + } + }) + + it("shrinks the budget and the outer kill in the invocation it actually makes", () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-manifest-wiring-")) + const reportRoot = path.join(repo, "reports") + const recordPath = path.join(repo, "invocations.jsonl") + fs.mkdirSync(path.join(repo, "packages", "core", "src"), { recursive: true }) + fs.mkdirSync(path.join(repo, "packages", "webview"), { recursive: true }) + const packageDir = path.join(repo, "node_modules", "@stryker-mutator", "core") + fs.mkdirSync(path.join(packageDir, "bin"), { recursive: true }) + fs.writeFileSync( + path.join(packageDir, "package.json"), + JSON.stringify({ name: "@stryker-mutator/core", bin: { stryker: "./bin/stryker.js" } }), + ) + fs.writeFileSync( + path.join(packageDir, "bin", "stryker.js"), + [ + 'const fs = require("node:fs")', + "const args = process.argv.slice(2)", + 'const kind = args.includes("--dryRunOnly") ? "preflight" : "run"', + 'fs.appendFileSync(process.env.U917_STRYKER_RECORD, JSON.stringify({ kind, args }) + "\\n")', + 'if (kind === "preflight") console.log("Instrumented 1 source file(s) with 1 mutant(s)")', + "process.exit(0)", + ].join("\n"), + ) + const previousRecord = process.env.U917_STRYKER_RECORD + const previousLog = console.log + const previousWarn = console.warn + const logs = [] + process.env.U917_STRYKER_RECORD = recordPath + console.log = (line) => logs.push(String(line)) + console.warn = () => {} + const clockValues = [0, 0, 9 * 60_000, 18 * 60_000, 27 * 60_000] + let tick = 0 + let failure + try { + runManifest( + repo, + { packages: [packageEntry("core", "packages/core"), packageEntry("webview", "packages/webview")] }, + reportRoot, + { + readMutationReport: () => ({ files: {} }), + now: () => { + const value = clockValues[tick] + assert.ok(tick < clockValues.length, "the injected clock ran out of readings") + tick += 1 + return value + }, + }, + ) + } catch (error) { + failure = error + } + try { + assert.equal(failure?.manifestBudgetStop, true) + const invocations = fs + .readFileSync(recordPath, "utf8") + .split("\n") + .filter((line) => line.trim()) + .map((line) => JSON.parse(line)) + assert.deepEqual( + invocations.map((call) => call.kind), + ["preflight", "run", "preflight"], + logs.join(" | ").slice(0, 400), + ) + const budgets = invocations.map((call) => call.args[call.args.indexOf("--dryRunTimeoutMinutes") + 1]) + assert.deepEqual(budgets, ["20", "13", "4"], JSON.stringify(budgets)) + const logged = logs.join("\n") + for (const kill of ["outer kill 25 min", "outer kill 18 min", "outer kill 9 min"]) { + assert.ok(logged.includes(kill), logged.slice(0, 400)) + } + for (const budget of [ + "initial-test-run budget 20 min (engine schema default 5 min)", + "initial-test-run budget 13 min (engine schema default 5 min)", + "initial-test-run budget 4 min (engine schema default 5 min)", + ]) { + assert.ok(logged.includes(budget), logged.slice(0, 400)) + } + assert.ok(failure.message.includes("was not launched"), failure.message) + } finally { + if (previousRecord === undefined) delete process.env.U917_STRYKER_RECORD + else process.env.U917_STRYKER_RECORD = previousRecord + console.log = previousLog + console.warn = previousWarn + fs.rmSync(repo, { recursive: true, force: true }) + } + }) +}) + +describe("suite summary isolation", () => { + it("keeps the job summary the suite inherited free of what the suite published", () => { + if (inheritedJobSummary === undefined) { + // A local run has no step summary to protect. The isolation still ran, and there is nothing + // inherited to read, so the assertion covers the file the suite gave itself instead. + assert.equal(process.env.GITHUB_STEP_SUMMARY, suiteSummary) + assert.ok(fs.existsSync(suiteSummary)) + return + } + const inherited = fs.readFileSync(inheritedJobSummary, "utf8") + assert.equal( + inherited, + "", + "the suite published into the job summary of the step that is running it: " + inherited.slice(0, 200), + ) + }) +}) From 1d6eb9812dab82e53ef18f54c5da926986e2a3ee Mon Sep 17 00:00:00 2001 From: "user.email" <136036952+easonLiangWorldedtech@users.noreply.github.com> Date: Sun, 15 Nov 2026 04:02:32 +0800 Subject: [PATCH 3/3] fix(gate): cap related-test discovery by the manifest budget and publish rows before a budget stop Discovery ran ahead of the per-package budget check with its own five-minute spawn timeout, so a package that reached the gate near the end of the manifest budget could still spend five minutes before anything recorded the stop. Discovery is now refused when the budget minus the terminus reserve has nothing left, and capped at what it has when it has less than five minutes. A budget stop rethrew out of the per-package catch, so appendSummary never saw the rows for the packages that had already finished and the job summary stayed empty. The rows are published first; main still writes the rejection record in its own catch as a separate outcome. Both directions were run on scripts/stryker-diff.test.mjs with node --test: head script plus the two new cases fails 2 of 127, fixed script plus the two new cases passes 127 of 127. --- scripts/stryker-diff.mjs | 41 ++++++++-- scripts/stryker-diff.test.mjs | 141 ++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 5 deletions(-) diff --git a/scripts/stryker-diff.mjs b/scripts/stryker-diff.mjs index 5e61f6a51d..19d7f7d32a 100644 --- a/scripts/stryker-diff.mjs +++ b/scripts/stryker-diff.mjs @@ -39,6 +39,15 @@ export const JOB_TIMEOUT_MINUTES = 30 export const TERMINUS_RESERVE_MINUTES = 3 export const MANIFEST_BUDGET_MINUTES = JOB_TIMEOUT_MINUTES - TERMINUS_RESERVE_MINUTES export const manifestBudgetMs = MANIFEST_BUDGET_MINUTES * 60 * 1_000 +// Related-test discovery runs before the engine is handed a budget, so it needs its own cap: the +// time the manifest still has left after TERMINUS_RESERVE_MINUTES, which is what recording a stop and +// appending the job summary costs. A cap of zero means discovery is refused before it starts. +export const DISCOVERY_TIMEOUT_MINUTES = 5 +export const discoveryTimeoutMs = (spentMinutes, budgetMinutes = MANIFEST_BUDGET_MINUTES) => { + const spareMinutes = budgetMinutes - spentMinutes - TERMINUS_RESERVE_MINUTES + if (spareMinutes <= 0) return 0 + return Math.min(DISCOVERY_TIMEOUT_MINUTES, spareMinutes) * 60 * 1_000 +} export function manifestRunBudgetMinutes(spentMinutes, budgetMinutes = MANIFEST_BUDGET_MINUTES) { const spareMinutes = budgetMinutes - spentMinutes @@ -462,7 +471,12 @@ export function resolveStrykerTempDir(repoRoot, runRoot) { return path.relative(runRoot, path.join(repoRoot, ".stryker-tmp")) || ".stryker-tmp" } -export function discoverRelatedTestFiles(repoRoot, packageEntry, reportDirectory) { +export function discoverRelatedTestFiles( + repoRoot, + packageEntry, + reportDirectory, + timeoutMs = DISCOVERY_TIMEOUT_MINUTES * 60 * 1_000, +) { const packageRoot = path.join(repoRoot, packageEntry.root) const runRoot = path.join(repoRoot, packageEntry.runRoot ?? packageEntry.root) const outputFile = path.join(reportDirectory, "vitest-related.json") @@ -474,14 +488,14 @@ export function discoverRelatedTestFiles(repoRoot, packageEntry, reportDirectory { cwd: runRoot, encoding: "utf8", - timeout: 5 * 60 * 1_000, + timeout: timeoutMs, maxBuffer: 50 * 1024 * 1024, env: process.env, }, ) if (result.error?.code === "ETIMEDOUT") { - throw new Error(`${packageEntry.id} related-test discovery exceeded 5 minutes`) + throw new Error(`${packageEntry.id} related-test discovery exceeded ${Math.round(timeoutMs / 60_000)} minutes`) } if (result.error) { throw new Error(`${packageEntry.id} related-test discovery could not start: ${result.error.message}`) @@ -869,6 +883,7 @@ export function runManifest( { runMutation = runStryker, readMutationReport = (reportPath) => JSON.parse(fs.readFileSync(reportPath, "utf8")), + discoverTests = discoverRelatedTestFiles, now = () => Date.now(), budgetMinutes = MANIFEST_BUDGET_MINUTES, } = {}, @@ -893,7 +908,17 @@ export function runManifest( const reportDirectory = path.join(reportRoot, packageEntry.id) fs.mkdirSync(reportDirectory, { recursive: true }) if (packageEntry.discoverRelatedTests) { - packageEntry.testFiles = discoverRelatedTestFiles(repoRoot, packageEntry, reportDirectory) + const spentBeforeDiscovery = (now() - manifestStartMs) / 60_000 + const discoveryMs = discoveryTimeoutMs(spentBeforeDiscovery, budgetMinutes) + if (discoveryMs === 0) { + throw manifestBudgetError( + packageEntry, + "related-test discovery", + spentBeforeDiscovery, + budgetMinutes, + ) + } + packageEntry.testFiles = discoverTests(repoRoot, packageEntry, reportDirectory, discoveryMs) } const preflightOutput = stripAnsi( runMutation(repoRoot, packageEntry, reportRoot, true, launchBudgetMinutes(packageEntry, "preflight")), @@ -952,7 +977,13 @@ export function runManifest( result: counts.blocking.length > 0 || counts.advisories.length > 0 ? "Advisory findings" : "Passed", }) } catch (error) { - if (error?.manifestBudgetStop === true) throw error + if (error?.manifestBudgetStop === true) { + // The budget stop ends the run itself, so the summary has to carry the packages that did + // finish before the rejection travels to main; main keeps writing the rejection record as a + // separate outcome in its own catch. + appendSummary(rows, advisories, manifest, path.join(reportRoot, PR_COMMENT_FILENAME)) + throw error + } advisories.push(error.message) rows.push({ id: packageEntry.id, diff --git a/scripts/stryker-diff.test.mjs b/scripts/stryker-diff.test.mjs index 65ae30b82f..45308d7dbc 100644 --- a/scripts/stryker-diff.test.mjs +++ b/scripts/stryker-diff.test.mjs @@ -2360,6 +2360,147 @@ describe("manifest wall-clock budget", () => { }) }) +describe("budget stop publishing and discovery cap", () => { + const packageEntry = (id, root, discoverRelatedTests) => ({ + id, + root, + vitestConfig: "vitest.unit.config.ts", + selectors: ["src/value.ts:1-1"], + changedExecutableLines: 1, + ...(discoverRelatedTests === undefined ? {} : { discoverRelatedTests }), + }) + const preflightLine = "Instrumented 1 source file(s) with 1 mutant(s)" + + it("publishes the finished package before a later budget stop is rethrown", () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-stop-publish-")) + const reportRoot = path.join(repo, "reports") + fs.mkdirSync(reportRoot, { recursive: true }) + const previousSummary = process.env.GITHUB_STEP_SUMMARY + const suiteSummaryPath = path.join(repo, "step-summary.md") + fs.writeFileSync(suiteSummaryPath, "") + process.env.GITHUB_STEP_SUMMARY = suiteSummaryPath + const clockValues = [0, 0, 10 * 60_000, 26 * 60_000] + let tick = 0 + let failure + try { + runManifest( + repo, + { + packages: [packageEntry("core", "packages/core"), packageEntry("webview", "packages/webview")], + headSha: "c".repeat(40), + }, + reportRoot, + { + runMutation: (_repoRoot, _entry, _reportRoot, dryRunOnly) => (dryRunOnly ? preflightLine : ""), + readMutationReport: () => ({ files: {} }), + now: () => { + const value = clockValues[tick] + assert.ok(tick < clockValues.length, "the injected clock ran out of readings") + tick += 1 + return value + }, + }, + ) + } catch (error) { + failure = error + } + try { + assert.ok(failure, "the manifest should have refused the second preflight") + assert.equal(failure.manifestBudgetStop, true) + const commentPath = path.join(reportRoot, PR_COMMENT_FILENAME) + assert.ok( + fs.existsSync(commentPath), + "a budget stop threw away the row for the package that had already finished", + ) + const comment = fs.readFileSync(commentPath, "utf8") + assert.ok(comment.includes("| core |"), comment.slice(0, 400)) + assert.ok(!comment.includes("| webview |"), comment.slice(0, 400)) + const published = fs.readFileSync(suiteSummaryPath, "utf8") + assert.ok(published.includes("| core |"), published.slice(0, 400)) + const rejectionRoot = path.join(repo, "rejection") + recordRejection(rejectionRoot, "b".repeat(40), "c".repeat(40), failure) + const recorded = fs.readFileSync(path.join(rejectionRoot, REJECTION_FILENAME), "utf8") + assert.ok(recorded.includes("was not launched"), recorded.slice(0, 240)) + assert.ok(!recorded.includes("| core |"), recorded.slice(0, 240)) + } finally { + if (previousSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY + else process.env.GITHUB_STEP_SUMMARY = previousSummary + fs.rmSync(repo, { recursive: true, force: true }) + } + }) + + it("caps related-test discovery at the budget that is left and refuses it when none is", () => { + const caps = [] + const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-discovery-cap-")) + const reportRoot = path.join(repo, "reports") + const previousSummary = process.env.GITHUB_STEP_SUMMARY + const summaryFile = path.join(repo, "step-summary.md") + fs.writeFileSync(summaryFile, "") + process.env.GITHUB_STEP_SUMMARY = summaryFile + try { + fs.mkdirSync(reportRoot, { recursive: true }) + const clockValues = [0, 20.5 * 60_000, 21 * 60_000] + let tick = 0 + runManifest( + repo, + { packages: [packageEntry("core", "packages/core", true)], headSha: "c".repeat(40) }, + reportRoot, + { + discoverTests: (_repoRoot, _entry, _reportRoot, timeoutMs) => { + caps.push(timeoutMs) + return ["src/value.test.ts"] + }, + runMutation: (_repoRoot, _entry, _reportRoot, dryRunOnly) => (dryRunOnly ? preflightLine : ""), + readMutationReport: () => ({ files: {} }), + now: () => { + const value = clockValues[tick] + assert.ok(tick < clockValues.length, "the injected clock ran out of readings") + tick += 1 + return value + }, + }, + ) + assert.deepEqual(caps, [(MANIFEST_BUDGET_MINUTES - 20.5 - 3) * 60_000], JSON.stringify(caps)) + assert.ok(caps[0] < 5 * 60_000, "discovery kept its full five minutes with 6.5 min of budget left") + const refusedRoot = path.join(repo, "refused") + fs.mkdirSync(path.join(refusedRoot, "reports"), { recursive: true }) + const refusals = [] + const refusalClock = [0, 24.5 * 60_000] + let refusalTick = 0 + let failure + try { + runManifest( + repo, + { packages: [packageEntry("core", "packages/core", true)], headSha: "c".repeat(40) }, + path.join(refusedRoot, "reports"), + { + discoverTests: (_repoRoot, _entry, _reportRoot, timeoutMs) => { + refusals.push(timeoutMs) + return ["src/value.test.ts"] + }, + now: () => { + const value = refusalClock[refusalTick] + assert.ok(refusalTick < refusalClock.length, "the refusal clock ran out of readings") + refusalTick += 1 + return value + }, + }, + ) + } catch (error) { + failure = error + } + assert.ok(failure, "discovery started although only 2.5 min of the budget was left") + assert.equal(failure.manifestBudgetStop, true) + assert.ok(failure.message.includes("related-test discovery was not launched"), failure.message) + assert.deepEqual(refusals, [], "discovery ran after the manifest said it had no room left") + } finally { + if (previousSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY + else process.env.GITHUB_STEP_SUMMARY = previousSummary + fs.rmSync(repo, { recursive: true, force: true }) + } + }) +}) + describe("suite summary isolation", () => { it("keeps the job summary the suite inherited free of what the suite published", () => { if (inheritedJobSummary === undefined) {