|
| 1 | +import { spawnSync } from "node:child_process" |
| 2 | +import { |
| 3 | + appendFileSync, |
| 4 | + closeSync, |
| 5 | + existsSync, |
| 6 | + mkdirSync, |
| 7 | + openSync, |
| 8 | + readFileSync, |
| 9 | + readdirSync, |
| 10 | + writeFileSync, |
| 11 | +} from "node:fs" |
| 12 | +import { createRequire } from "node:module" |
| 13 | +import os from "node:os" |
| 14 | +import path from "node:path" |
| 15 | +import { performance } from "node:perf_hooks" |
| 16 | +import { fileURLToPath } from "node:url" |
| 17 | +import { parseArgs } from "node:util" |
| 18 | + |
| 19 | +const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") |
| 20 | + |
| 21 | +export function validateOptions(workers, repetitions) { |
| 22 | + if (![1, 2, 4].includes(workers) || !Number.isInteger(repetitions) || repetitions < 1 || repetitions > 5) { |
| 23 | + throw new Error("Workers must be 1, 2 or 4; repetitions must be an integer from 1 to 5") |
| 24 | + } |
| 25 | +} |
| 26 | + |
| 27 | +export function summarizeReport(report) { |
| 28 | + const inventory = report.files |
| 29 | + .map((file) => [file.file, file.tests.map((test) => test.name).sort()]) |
| 30 | + .sort(([left], [right]) => left.localeCompare(right, "en")) |
| 31 | + const tests = report.files.flatMap((file) => file.tests) |
| 32 | + return { |
| 33 | + inventory, |
| 34 | + fileCount: report.files.length, |
| 35 | + testCount: tests.length, |
| 36 | + passed: tests.filter((test) => test.state === "passed").length, |
| 37 | + skipped: tests.filter((test) => test.state === "skipped" || test.state === "pending").length, |
| 38 | + failed: tests.filter((test) => test.state === "failed").length, |
| 39 | + failedFiles: report.files.filter((file) => !file.ok).length, |
| 40 | + unhandledErrors: report.errors.length, |
| 41 | + retries: tests.reduce((count, test) => count + (test.diagnostic?.retryCount ?? 0), 0), |
| 42 | + } |
| 43 | +} |
| 44 | + |
| 45 | +export function coveredLines(lcov) { |
| 46 | + let source |
| 47 | + const covered = new Set() |
| 48 | + for (const line of lcov.split(/\r?\n/)) { |
| 49 | + if (line.startsWith("SF:")) source = line.slice(3).replaceAll("\\", "/") |
| 50 | + if (line.startsWith("DA:")) { |
| 51 | + const [number, hits] = line.slice(3).split(",") |
| 52 | + if ( |
| 53 | + !source || |
| 54 | + !Number.isInteger(Number(number)) || |
| 55 | + Number(number) < 1 || |
| 56 | + !Number.isInteger(Number(hits)) || |
| 57 | + Number(hits) < 0 |
| 58 | + ) { |
| 59 | + throw new Error("Invalid LCOV line record") |
| 60 | + } |
| 61 | + if (Number(hits) > 0) covered.add(`${source}:${number}`) |
| 62 | + } |
| 63 | + if (line === "end_of_record") source = undefined |
| 64 | + } |
| 65 | + return [...covered].sort() |
| 66 | +} |
| 67 | + |
| 68 | +export function compareRuns(runs, repetitions = 2) { |
| 69 | + const errors = [] |
| 70 | + const inventory = JSON.stringify(runs[0]?.inventory) |
| 71 | + const sha = runs[0]?.sha |
| 72 | + const linuxCoverage = runs.find((run) => run.platform === "linux")?.coveredLines |
| 73 | + if (runs.length !== 6 * repetitions) errors.push(`Expected ${6 * repetitions} benchmark runs, got ${runs.length}`) |
| 74 | + for (const platform of ["linux", "win32"]) { |
| 75 | + for (const workers of [1, 2, 4]) { |
| 76 | + const group = runs.filter((run) => run.platform === platform && run.workers === workers) |
| 77 | + if ( |
| 78 | + group.length !== repetitions || |
| 79 | + new Set(group.map((run) => run.repetition)).size !== repetitions || |
| 80 | + group.some( |
| 81 | + (run) => !Number.isInteger(run.repetition) || run.repetition < 1 || run.repetition > repetitions, |
| 82 | + ) |
| 83 | + ) { |
| 84 | + errors.push(`${platform}/${workers}: expected ${repetitions} distinct repetitions, got ${group.length}`) |
| 85 | + } |
| 86 | + } |
| 87 | + } |
| 88 | + for (const run of runs) { |
| 89 | + const label = `${run.platform}/${run.workers}/r${run.repetition}` |
| 90 | + if ( |
| 91 | + run.exitCode !== 0 || |
| 92 | + run.failed || |
| 93 | + run.failedFiles || |
| 94 | + run.unhandledErrors || |
| 95 | + run.retries || |
| 96 | + !run.testCount || |
| 97 | + run.passed + run.skipped !== run.testCount |
| 98 | + ) { |
| 99 | + errors.push(`${label}: incomplete, failing or retried tests`) |
| 100 | + } |
| 101 | + if (run.sha !== sha || JSON.stringify(run.inventory) !== inventory) |
| 102 | + errors.push(`${label}: source/test inventory differs`) |
| 103 | + if (run.coverage !== (run.platform === "linux")) errors.push(`${label}: coverage mode differs from the CI lane`) |
| 104 | + if ( |
| 105 | + run.platform === "linux" && |
| 106 | + (!run.coveredLines?.length || JSON.stringify(run.coveredLines) !== JSON.stringify(linuxCoverage)) |
| 107 | + ) { |
| 108 | + errors.push(`${label}: covered-line identities differ or coverage is missing`) |
| 109 | + } |
| 110 | + } |
| 111 | + return errors |
| 112 | +} |
| 113 | + |
| 114 | +function findMetadata(directory) { |
| 115 | + if (!existsSync(directory)) return [] |
| 116 | + return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { |
| 117 | + const fullPath = path.join(directory, entry.name) |
| 118 | + return entry.isDirectory() ? findMetadata(fullPath) : entry.name === "metadata.json" ? [fullPath] : [] |
| 119 | + }) |
| 120 | +} |
| 121 | + |
| 122 | +function publishSummary(markdown) { |
| 123 | + console.log(markdown) |
| 124 | + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown + "\n") |
| 125 | +} |
| 126 | + |
| 127 | +function summarize(output, repetitions) { |
| 128 | + const runs = findMetadata(output).map((filename) => { |
| 129 | + const metadata = JSON.parse(readFileSync(filename, "utf8")) |
| 130 | + if (metadata.coverage) { |
| 131 | + const coverageFile = path.join(path.dirname(filename), "coverage/lcov.info") |
| 132 | + metadata.coveredLines = existsSync(coverageFile) ? coveredLines(readFileSync(coverageFile, "utf8")) : [] |
| 133 | + } |
| 134 | + return metadata |
| 135 | + }) |
| 136 | + const errors = compareRuns(runs, repetitions) |
| 137 | + mkdirSync(output, { recursive: true }) |
| 138 | + writeFileSync(path.join(output, "comparison.json"), JSON.stringify({ errors, runs }, null, 2) + "\n") |
| 139 | + const table = [ |
| 140 | + "## Webview benchmark", |
| 141 | + "", |
| 142 | + "Elapsed time measures the Vitest process, including startup and coverage reporting; installation and prerequisite builds are separate steps.", |
| 143 | + "", |
| 144 | + "| Platform | Workers | Repetition | Coverage | Seconds | Files | Tests | Exit |", |
| 145 | + "| --- | --- | --- | --- | --- | --- | --- | --- |", |
| 146 | + ...runs.map( |
| 147 | + (run) => |
| 148 | + `| ${run.platform} | ${run.workers} | ${run.repetition} | ${run.coverage} | ${run.wallSeconds.toFixed(2)} | ${run.fileCount} | ${run.testCount} | ${run.exitCode} |`, |
| 149 | + ), |
| 150 | + "", |
| 151 | + ...errors.map((error) => `- ${error}`), |
| 152 | + ] |
| 153 | + publishSummary(table.join("\n")) |
| 154 | + if (errors.length) process.exitCode = 1 |
| 155 | +} |
| 156 | + |
| 157 | +function measure(workers, repetitions, output) { |
| 158 | + validateOptions(workers, repetitions) |
| 159 | + const webviewRoot = path.join(repositoryRoot, "webview-ui") |
| 160 | + const require = createRequire(path.join(webviewRoot, "package.json")) |
| 161 | + const vitest = path.join(path.dirname(require.resolve("vitest/package.json")), "vitest.mjs") |
| 162 | + const coverage = process.env.WEBVIEW_BENCHMARK_COVERAGE === "true" |
| 163 | + for (let repetition = 1; repetition <= repetitions; repetition++) { |
| 164 | + const runDirectory = path.join(output, `run-${repetition}`) |
| 165 | + mkdirSync(runDirectory, { recursive: true }) |
| 166 | + if (existsSync(path.join(runDirectory, "metadata.json"))) |
| 167 | + throw new Error(`Refusing to overwrite ${runDirectory}`) |
| 168 | + const stdout = openSync(path.join(runDirectory, "stdout.log"), "w") |
| 169 | + const stderr = openSync(path.join(runDirectory, "stderr.log"), "w") |
| 170 | + const args = [ |
| 171 | + vitest, |
| 172 | + "run", |
| 173 | + "--config", |
| 174 | + "vitest.benchmark.config.ts", |
| 175 | + `--maxWorkers=${workers}`, |
| 176 | + "--pool=forks", |
| 177 | + "--isolate", |
| 178 | + "--logHeapUsage", |
| 179 | + "--reporter=default", |
| 180 | + `--reporter=${path.join(repositoryRoot, "scripts/ci/webview-benchmark-reporter.mjs")}`, |
| 181 | + ] |
| 182 | + if (coverage) args.push("--coverage") |
| 183 | + console.log(`::group::${process.platform}: ${workers} workers, repetition ${repetition}, coverage=${coverage}`) |
| 184 | + const startedAt = new Date().toISOString() |
| 185 | + const start = performance.now() |
| 186 | + let result |
| 187 | + try { |
| 188 | + result = spawnSync(process.execPath, args, { |
| 189 | + cwd: webviewRoot, |
| 190 | + env: { |
| 191 | + ...process.env, |
| 192 | + CI: "true", |
| 193 | + TZ: "UTC", |
| 194 | + WEBVIEW_BENCHMARK_RUN_DIR: runDirectory, |
| 195 | + WEBVIEW_BENCHMARK_REPORT: path.join(runDirectory, "modules.json"), |
| 196 | + }, |
| 197 | + stdio: ["ignore", stdout, stderr], |
| 198 | + timeout: 15 * 60 * 1000, |
| 199 | + }) |
| 200 | + } finally { |
| 201 | + closeSync(stdout) |
| 202 | + closeSync(stderr) |
| 203 | + } |
| 204 | + const wallSeconds = (performance.now() - start) / 1000 |
| 205 | + const reportFile = path.join(runDirectory, "modules.json") |
| 206 | + const summary = existsSync(reportFile) |
| 207 | + ? summarizeReport(JSON.parse(readFileSync(reportFile, "utf8"))) |
| 208 | + : { fileCount: 0, testCount: 0, inventory: [] } |
| 209 | + const exitCode = result.status ?? 1 |
| 210 | + const metadata = { |
| 211 | + ...summary, |
| 212 | + workers, |
| 213 | + repetition, |
| 214 | + coverage, |
| 215 | + exitCode, |
| 216 | + wallSeconds, |
| 217 | + startedAt, |
| 218 | + completedAt: new Date().toISOString(), |
| 219 | + sha: |
| 220 | + process.env.GITHUB_SHA ?? |
| 221 | + spawnSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8" }).stdout.trim(), |
| 222 | + platform: process.platform, |
| 223 | + arch: process.arch, |
| 224 | + node: process.version, |
| 225 | + vitest: require("vitest/package.json").version, |
| 226 | + availableParallelism: os.availableParallelism(), |
| 227 | + cpuModel: os.cpus()[0]?.model, |
| 228 | + totalMemoryBytes: os.totalmem(), |
| 229 | + imageOS: process.env.ImageOS, |
| 230 | + imageVersion: process.env.ImageVersion, |
| 231 | + runId: process.env.GITHUB_RUN_ID, |
| 232 | + runAttempt: process.env.GITHUB_RUN_ATTEMPT, |
| 233 | + error: result.error?.message, |
| 234 | + signal: result.signal, |
| 235 | + } |
| 236 | + writeFileSync(path.join(runDirectory, "metadata.json"), JSON.stringify(metadata, null, 2) + "\n") |
| 237 | + console.log(readFileSync(path.join(runDirectory, "stdout.log"), "utf8")) |
| 238 | + console.error(readFileSync(path.join(runDirectory, "stderr.log"), "utf8")) |
| 239 | + console.log("::endgroup::") |
| 240 | + publishSummary( |
| 241 | + `Repetition ${repetition}: ${workers} workers, ${wallSeconds.toFixed(2)}s, ${summary.testCount} tests, exit ${exitCode}.`, |
| 242 | + ) |
| 243 | + if ( |
| 244 | + exitCode !== 0 || |
| 245 | + !summary.testCount || |
| 246 | + summary.failed || |
| 247 | + summary.failedFiles || |
| 248 | + summary.unhandledErrors || |
| 249 | + summary.retries |
| 250 | + ) |
| 251 | + process.exitCode = 1 |
| 252 | + } |
| 253 | +} |
| 254 | + |
| 255 | +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { |
| 256 | + const { values } = parseArgs({ |
| 257 | + options: { |
| 258 | + workers: { type: "string" }, |
| 259 | + repetitions: { type: "string", default: "2" }, |
| 260 | + output: { type: "string", default: "benchmark-results" }, |
| 261 | + summarize: { type: "boolean", default: false }, |
| 262 | + }, |
| 263 | + }) |
| 264 | + const output = path.resolve(values.output) |
| 265 | + const repetitions = Number(values.repetitions) |
| 266 | + if (values.summarize) { |
| 267 | + validateOptions(1, repetitions) |
| 268 | + summarize(output, repetitions) |
| 269 | + } else { |
| 270 | + measure(Number(values.workers), repetitions, output) |
| 271 | + } |
| 272 | +} |
0 commit comments