ev-dev-labs
diff --git a/‎.github/scripts/check_vitest_blobs.mjs‎
Lines changed: 43 additions & 0 deletions b/‎.github/scripts/check_vitest_blobs.mjs‎
Lines changed: 43 additions & 0 deletions
diff --git a/‎.github/scripts/check_vitest_blobs.test.mjs‎
Lines changed: 52 additions & 0 deletions b/‎.github/scripts/check_vitest_blobs.test.mjs‎
Lines changed: 52 additions & 0 deletions
diff --git a/‎.github/scripts/ci_test_shards.py‎
Lines changed: 113 additions & 0 deletions b/‎.github/scripts/ci_test_shards.py‎
Lines changed: 113 additions & 0 deletions
diff --git a/‎.github/scripts/ci_workflows.test.mjs‎
Lines changed: 82 additions & 0 deletions b/‎.github/scripts/ci_workflows.test.mjs‎
Lines changed: 82 additions & 0 deletions
@@ -0,0 +1,43 @@
+import { readFileSync, readdirSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+// Decode with Vitest's own dependency, including when npm does not hoist it.
+const requireWeb = createRequire(new URL('../../web/package.json', import.meta.url));
+const requireVitest = createRequire(requireWeb.resolve('vitest/package.json'));
+const { parse } = requireVitest('flatted');
+
+export function checkBlobs(directory, count, expectedFiles) {
+  if (!Number.isInteger(count) || count < 1) throw new Error('Invalid shard count');
+  const expectedNames = Array.from({ length: count }, (_, i) => `blob-${i + 1}-${count}.json`).sort();
+  if (JSON.stringify(readdirSync(directory).sort()) !== JSON.stringify(expectedNames)) {
+    throw new Error('Missing or unexpected Vitest shard artifacts');
+  }
+  const files = new Set();
+  for (const name of expectedNames) {
+    const [version, modules, errors, coverage] = parse(readFileSync(join(directory, name), 'utf8'));
+    if (!version || !Array.isArray(modules) || modules.length === 0
+      || !Array.isArray(errors) || errors.length > 0
+      || !coverage || Object.keys(coverage).length === 0) {
+      throw new Error(`${name}: missing tests, missing coverage, or unhandled errors`);
+    }
+    for (const module of modules) {
+      if (typeof module.filepath !== 'string' || files.has(module.filepath)) {
+        throw new Error(`${name}: invalid or duplicated test file`);
+      }
+      files.add(module.filepath);
+    }
+  }
+  const expected = new Set(expectedFiles);
+  if (files.size !== expected.size || [...expected].some(file => !files.has(file))) {
+    throw new Error('Shard test files differ from full Vitest discovery');
+  }
+  console.log(`Validated ${count} Vitest shards: ${files.size} test files, no gaps or overlaps`);
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  const [, , directory, count, discovery] = process.argv;
+  const files = JSON.parse(readFileSync(discovery, 'utf8')).map(entry => entry.file);
+  checkBlobs(directory, Number(count), files);
+}
@@ -0,0 +1,52 @@
+import assert from 'node:assert/strict';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, test } from 'node:test';
+import { checkBlobs } from './check_vitest_blobs.mjs';
+
+const requireWeb = createRequire(new URL('../../web/package.json', import.meta.url));
+const { stringify } = createRequire(requireWeb.resolve('vitest/package.json'))('flatted');
+let root;
+const blob = (files, coverage = { 'source.ts': {} }, errors = []) =>
+  stringify(['4.1.2', files.map(filepath => ({ filepath })), errors, coverage]);
+
+beforeEach(() => {
+  root = mkdtempSync(join(tmpdir(), 'vitest-shards-'));
+  writeFileSync(join(root, 'blob-1-2.json'), blob(['/a.test.ts']));
+  writeFileSync(join(root, 'blob-2-2.json'), blob(['/b.test.ts']));
+});
+afterEach(() => rmSync(root, { recursive: true, force: true }));
+
+test('accepts exact discovery across all shards regardless of order', () => {
+  checkBlobs(root, 2, ['/b.test.ts', '/a.test.ts']);
+});
+test('rejects missing and extraneous artifacts', () => {
+  assert.throws(() => checkBlobs(root, 3, []), /artifacts/);
+  writeFileSync(join(root, 'extra.json'), blob(['/c.test.ts']));
+  assert.throws(() => checkBlobs(root, 2, []), /artifacts/);
+});
+test('rejects duplicated test files even if the artifact count is correct', () => {
+  writeFileSync(join(root, 'blob-2-2.json'), blob(['/a.test.ts']));
+  assert.throws(() => checkBlobs(root, 2, ['/a.test.ts', '/b.test.ts']), /duplicated/);
+});
+test('rejects omitted and unexpected tests', () => {
+  for (const expected of [['/a.test.ts'], ['/a.test.ts', '/b.test.ts', '/c.test.ts']]) {
+    assert.throws(() => checkBlobs(root, 2, expected), /discovery/);
+  }
+});
+test('rejects empty tests, missing coverage, and unhandled worker errors', () => {
+  for (const value of [blob([]), blob(['/b.test.ts'], null), blob(['/b.test.ts'], {}),
+    blob(['/b.test.ts'], { 'source.ts': {} }, [{ message: 'worker crashed' }])]) {
+    writeFileSync(join(root, 'blob-2-2.json'), value);
+    assert.throws(() => checkBlobs(root, 2, ['/a.test.ts', '/b.test.ts']), /missing tests/);
+  }
+});
+test('rejects corrupt reports and invalid shard counts', () => {
+  writeFileSync(join(root, 'blob-2-2.json'), '{broken');
+  assert.throws(() => checkBlobs(root, 2, []), SyntaxError);
+  for (const count of [0, -1, 1.5, NaN]) {
+    assert.throws(() => checkBlobs(root, count, []), /count/);
+  }
+});
@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""Partition every Go package and merge complete, validated race/coverage shards."""
+
+import argparse
+import json
+import re
+import subprocess
+from pathlib import Path
+
+
+def discover():
+    output = subprocess.check_output(["go", "list", "-race", "-json", "./..."], text=True, encoding="utf-8")
+    decoder = json.JSONDecoder()
+    packages = []
+    while output.strip():
+        package, end = decoder.raw_decode(output.lstrip())
+        output = output.lstrip()[end:]
+        files = package.get("TestGoFiles", []) + package.get("XTestGoFiles", [])
+        weight = 1 + sum((Path(package["Dir"]) / name).stat().st_size for name in files)
+        packages.append((package["ImportPath"], weight))
+    return packages
+
+
+def partition(packages, count):
+    names = [name for name, _ in packages]
+    if count < 1 or count > len(names) or len(set(names)) != len(names):
+        raise ValueError("shards require distinct packages and a nonempty partition")
+    shards = [[] for _ in range(count)]
+    weights = [0] * count
+    # Large test packages start first; stable tie-breaking is identical on every runner.
+    for name, weight in sorted(packages, key=lambda item: (-item[1], item[0])):
+        index = min(range(count), key=lambda i: (weights[i], i))
+        shards[index].append(name)
+        weights[index] += weight
+    return [sorted(shard) for shard in shards]
+
+
+def merge(root, expected, output):
+    count = len(expected)
+    directories = {path.name for path in root.iterdir()}
+    if directories != {f"backend-test-{i}" for i in range(1, count + 1)}:
+        raise ValueError("missing or unexpected backend shard artifacts")
+    blocks = {}
+    events = []
+    for index, packages in enumerate(expected, 1):
+        directory = root / f"backend-test-{index}"
+        manifest = json.loads((directory / "manifest.json").read_text(encoding="utf-8"))
+        if manifest != {"index": index, "count": count, "packages": packages}:
+            raise ValueError(f"shard {index}: package manifest differs from discovery")
+        terminal = set()
+        for line in (directory / "test-events.json").read_text(encoding="utf-8").splitlines():
+            event = json.loads(line)
+            package = event.get("Package")
+            if package and package not in packages:
+                raise ValueError(f"shard {index}: unexpected test package {package}")
+            if event.get("Action") == "fail":
+                raise ValueError(f"shard {index}: failing test event")
+            if not event.get("Test") and event.get("Action") in {"pass", "skip"}:
+                terminal.add(package)
+            events.append(line)
+        if terminal != set(packages):
+            raise ValueError(f"shard {index}: incomplete package test events")
+        profile = (directory / "coverage.out").read_text(encoding="utf-8").splitlines()
+        if not profile or profile[0] != "mode: atomic":
+            raise ValueError(f"shard {index}: expected atomic coverage profile")
+        for line in profile[1:]:
+            match = re.fullmatch(r"(.+:\d+\.\d+,\d+\.\d+) (\d+) (\d+)", line)
+            if not match:
+                raise ValueError(f"shard {index}: malformed coverage block")
+            key, statements, hits = match.groups()
+            statements, hits = int(statements), int(hits)
+            if key in blocks and blocks[key][0] != statements:
+                raise ValueError(f"shard {index}: inconsistent coverage block {key}")
+            blocks[key] = (statements, hits + blocks.get(key, (0, 0))[1])
+    if not blocks:
+        raise ValueError("merged coverage is empty")
+    output.mkdir(parents=True, exist_ok=True)
+    (output / "coverage.out").write_text(
+        "mode: atomic\n"
+        + "".join(f"{key} {statements} {hits}\n" for key, (statements, hits) in sorted(blocks.items())),
+        encoding="utf-8",
+    )
+    (output / "test-events.json").write_text("\n".join(events) + "\n", encoding="utf-8")
+    print(f"Merged {count} shards: {sum(map(len, expected))} packages, {len(blocks)} coverage blocks")
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("command", choices=["plan", "merge"])
+    parser.add_argument("--count", type=int, required=True)
+    parser.add_argument("--index", type=int)
+    parser.add_argument("--input", type=Path)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    shards = partition(discover(), args.count)
+    if args.command == "merge":
+        if args.input is None:
+            parser.error("merge requires --input")
+        merge(args.input, shards, args.output)
+        return
+    if args.index is None or not 1 <= args.index <= args.count:
+        parser.error("plan requires --index in 1..count")
+    packages = shards[args.index - 1]
+    args.output.mkdir(parents=True, exist_ok=True)
+    (args.output / "packages.txt").write_text("\n".join(packages) + "\n", encoding="utf-8")
+    (args.output / "manifest.json").write_text(json.dumps({
+        "index": args.index, "count": args.count, "packages": packages,
+    }), encoding="utf-8")
+    print(f"Shard {args.index}/{args.count}: {len(packages)} packages")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,82 @@
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { test } from 'node:test';
+
+const require = createRequire(new URL('../../web/package.json', import.meta.url));
+const { load } = require('js-yaml');
+const workflow = name => load(readFileSync(new URL(`../workflows/${name}`, import.meta.url), 'utf8'));
+const ci = workflow('ci.yml');
+const browser = workflow('frontend-quality.yml');
+const commands = job => job.steps.map(step => step.run ?? '').join('\n');
+
+test('unit shard matrices match the strict merge counts and never fail-fast', () => {
+  for (const domain of ['backend', 'frontend']) {
+    const count = Number(ci.env[`${domain.toUpperCase()}_SHARDS`]);
+    const job = ci.jobs[`${domain}-tests`];
+    assert.deepEqual(job.strategy.matrix.shard, Array.from({ length: count }, (_, i) => i + 1));
+    assert.equal(job.strategy['fail-fast'], false);
+    assert.equal(job.needs, undefined);
+    assert.equal(ci.jobs[`${domain}-coverage`].needs, `${domain}-tests`);
+    assert.match(commands(ci.jobs[`${domain}-coverage`]), /ci_test_shards\.py merge|check_vitest_blobs\.mjs/);
+  }
+  assert.ok(ci.jobs['backend-tests'].services.postgres);
+  assert.ok(ci.jobs['backend-database'].services.postgres);
+  assert.match(commands(ci.jobs['backend-tests']), /go test -race .*covermode=atomic/);
+  assert.match(commands(ci.jobs['frontend-tests']), /--maxWorkers=2/);
+});
+
+test('existing aggregate names reject any required job that did not succeed', () => {
+  for (const [jobs, id, name, dependencies] of [
+    [ci.jobs, 'backend', 'Backend (lint + test + build)',
+      ['backend-checks', 'backend-tests', 'backend-coverage', 'backend-database', 'backend-build']],
+    [ci.jobs, 'frontend', 'Frontend (lint + test + build)',
+      ['frontend-checks', 'frontend-tests', 'frontend-coverage']],
+    [browser.jobs, 'chromium-quality', 'Chromium responsive, a11y, keyboard, and performance',
+      ['contract', 'browser-build', 'chromium-tests']],
+    [browser.jobs, 'visual', 'Deliberate visual snapshot gate (Windows baseline)',
+      ['contract', 'browser-build', 'visual-tests']],
+  ]) {
+    const job = jobs[id];
+    assert.equal(job.name, name);
+    assert.equal(job.if, 'always()');
+    assert.deepEqual(job.needs, dependencies);
+    assert.match(commands(job), /all\(job\["result"\] == "success" for job in results\.values\(\)\)/);
+    assert.equal(job.steps[0].env.RESULTS, '${{ toJSON(needs) }}');
+  }
+});
+
+test('independent validation runs in parallel but Docker waits for successful gates', () => {
+  for (const id of ['generated', 'backend-checks', 'backend-build', 'backend-database', 'frontend-checks']) {
+    assert.equal(ci.jobs[id].needs, undefined);
+  }
+  assert.deepEqual(ci.jobs.docker.needs, ['generated', 'backend', 'frontend']);
+  assert.equal(ci.jobs.docker.if, undefined);
+  const build = ci.jobs.docker.steps.find(step => step.uses?.startsWith('docker/build-push-action'));
+  assert.equal(build.with.push, false);
+});
+
+test('browser shards reuse one build and retain isolated performance and Windows baselines', () => {
+  const shards = browser.jobs['chromium-tests'].strategy.matrix.include;
+  assert.deepEqual(shards.filter(shard => shard.suite === 'quality').map(shard => shard.shard),
+    ['1/4', '2/4', '3/4', '4/4']);
+  for (const suite of ['a11y', 'performance']) {
+    assert.deepEqual(shards.filter(shard => shard.suite === suite).map(shard => shard.shard), ['1/1']);
+  }
+  for (const id of ['chromium-tests', 'visual-tests', 'cross-browser']) {
+    const job = browser.jobs[id];
+    assert.equal(job.needs, 'browser-build');
+    assert.equal(job.strategy['fail-fast'], false);
+    assert.ok(job.steps.some(step => step.with?.name === 'e2e-app' && step.uses?.startsWith('actions/download-artifact')));
+    assert.equal(job.steps.find(step => step.env?.E2E_REUSE_BUILD)?.env.E2E_REUSE_BUILD, '1');
+  }
+  assert.equal(browser.jobs['visual-tests']['runs-on'], 'windows-latest');
+  assert.deepEqual(browser.jobs['visual-tests'].strategy.matrix.shard, [1, 2, 3, 4]);
+  assert.equal(Object.values(browser.jobs).filter(job => commands(job).includes('npm run e2e:build')).length, 1);
+});
+
+test('parallelization introduces no additional nonblocking test exceptions', () => {
+  const waivers = Object.entries(ci.jobs).flatMap(([job, config]) =>
+    config.steps.filter(step => step['continue-on-error']).map(step => [job, step.name]));
+  assert.deepEqual(waivers, [['backend-database', 'Integration test (telemetry replay)']]);
+});