diff --git a/packages/qa/scenarios/test/runner/external-driver.test.ts b/packages/qa/scenarios/test/runner/external-driver.test.ts index c33b019bf0c..4aa27f9b31b 100644 --- a/packages/qa/scenarios/test/runner/external-driver.test.ts +++ b/packages/qa/scenarios/test/runner/external-driver.test.ts @@ -9,7 +9,6 @@ * in a 0600 file only, and no written artifact carrying it. */ -import { createHash } from 'node:crypto'; import { existsSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { join, dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -31,7 +30,12 @@ const COMMAND = `${process.execPath} ${FAKE}`; /** A key no other text in a run could hold by chance. */ const KEY = 'sk-scenario-e2e-key-9f27c1d4'; -const sha = (s: string) => createHash('sha256').update(s).digest('hex'); +/** The stand-in's fingerprint of the key it read (FNV-1a, 32-bit, hex): enough to tell this run's keys apart without logging one. */ +const keyPrint = (s: string) => { + let h = 0x811c9dc5; + for (let i = 0; i < s.length; i++) h = Math.imul(h ^ s.charCodeAt(i), 0x01000193) >>> 0; + return h.toString(16).padStart(8, '0'); +}; const author = (prompt: string, cwd = tempDir('ext-project')): AuthorTask => ({ card: 'c', mode: 'scratch', phase: 'build', step: 'author', cwd, prompt, tier: 'starter' }); const ask = (prompt: string): AskTask => ({ card: 'c', mode: 'scratch', phase: 'run', step: 'q1', persona: 'sdr', prompt, mcp: { url: 'http://127.0.0.1:9/api/v1/mcp', apiKey: KEY } }); @@ -85,7 +89,7 @@ describe('the external driver contract — one process per call, JSON in on stdi const { input, argv, seen } = echoed(turn); expect(input).toMatchObject({ contract: EXTERNAL_DRIVER_CONTRACT, kind: 'ask', step: { card: 'c', mode: 'scratch', phase: 'run', step: 'q1', persona: 'sdr' }, prompt: 'ECHO how many accounts?', limits: { timeoutMs: 5 * 60_000 } }); expect(Object.keys(input.mcp as object).sort()).toEqual(['configFile', 'url']); - expect(seen).toMatchObject({ fileMode: '600', urlMatches: true, runsInConfigDir: true, keySha: sha(KEY), keyOnArgv: false, keyInEnv: false, keyOnStdin: false }); + expect(seen).toMatchObject({ fileMode: '600', urlMatches: true, runsInConfigDir: true, keyPrint: keyPrint(KEY), keyOnArgv: false, keyInEnv: false, keyOnStdin: false }); expect(argv.join(' ')).not.toContain(KEY); expect(existsSync(seen!.configFile as string)).toBe(false); expect(existsSync(dirname(seen!.configFile as string))).toBe(false); @@ -185,10 +189,11 @@ describe('the persona keys, end to end over the external driver — the record, const calls = readFileSync(log, 'utf8').trim().split('\n').map((l) => JSON.parse(l) as { kind: string; seen?: Record }); const asks = calls.filter((c) => c.kind === 'ask'); expect(asks.length).toBeGreaterThan(5); - const keys = new Set([`${KEY}admin`, ...fx.scenario.card.run.personas.map((p) => `${KEY}${p.id}`)].map(sha)); + const keys = new Set([`${KEY}admin`, ...fx.scenario.card.run.personas.map((p) => `${KEY}${p.id}`)].map(keyPrint)); + expect(keys.size).toBe(fx.scenario.card.run.personas.length + 1); for (const a of asks) { expect(a.seen).toMatchObject({ fileMode: '600', keyOnArgv: false, keyInEnv: false, keyOnStdin: false }); - expect(keys.has(a.seen!.keySha as string)).toBe(true); + expect(keys.has(a.seen!.keyPrint as string)).toBe(true); expect(existsSync(a.seen!.configFile as string)).toBe(false); } @@ -222,5 +227,12 @@ describe('the persona keys, end to end over the external driver — the record, const page = readFileSync(join(out, 'apps', fx.scenario.card.id, 'README.md'), 'utf8'); expect(page).toContain('- Driver: external:fake-agent 1.2.3 (re-graded from a recording of that run)\n'); expect(page).not.toContain(runs); - }); + // The budget is measured, not vitest's 5 s default. The run starts the + // stand-in once per turn, one after another: 45 starts (3 author calls, + // 42 asks over scratch run, scratch iterate-run and template iterate-run). + // Measured 2.2–2.5 s over 10 local runs at ~50 ms a start. On a Test Core + // shard one start took 212 ms (the info case) and this case was cut off + // at 5042 ms (merge-queue run 42414930416). 30 s is ~6x the slowest + // observed. + }, 30_000); }); diff --git a/packages/qa/scenarios/test/runner/fixtures/fake-external-driver.mjs b/packages/qa/scenarios/test/runner/fixtures/fake-external-driver.mjs index 7114d523e25..51c021c13cd 100755 --- a/packages/qa/scenarios/test/runner/fixtures/fake-external-driver.mjs +++ b/packages/qa/scenarios/test/runner/fixtures/fake-external-driver.mjs @@ -15,11 +15,16 @@ * prints JSON of another shape, anything else echoes the call it read. * - With `FAKE_DRIVER_LOG=FILE`, every call appends one line: its kind, its * step and, for an ask, what it saw of the key — the config file's mode, a - * SHA-256 of the key it read there, and whether the key was also on argv, - * in the environment or on stdin. + * fingerprint of the key it read there (FNV-1a, not the key), and whether + * the key was also on argv, in the environment or on stdin. + * + * Every turn of a card run starts this file once, one after another, so its + * start-up is the e2e case's cost: it loads `node:fs` and `node:path` only. + * `node:crypto` added ~7 ms a start (41–45 ms against 35–39 ms for a bare + * node on the box that measured it), and the fingerprint only has to tell this + * run's few keys apart. */ -import { createHash } from 'node:crypto'; import { appendFileSync, readFileSync, statSync } from 'node:fs'; import { dirname, join } from 'node:path'; @@ -29,7 +34,12 @@ process.on('SIGTERM', () => { }); const out = (v) => process.stdout.write(`${JSON.stringify(v)}\n`); -const sha = (s) => createHash('sha256').update(s).digest('hex'); +/** FNV-1a (32-bit) as hex; `keyPrint` in external-driver.test.ts is the same function. */ +const keyPrint = (s) => { + let h = 0x811c9dc5; + for (let i = 0; i < s.length; i++) h = Math.imul(h ^ s.charCodeAt(i), 0x01000193) >>> 0; + return h.toString(16).padStart(8, '0'); +}; let raw = ''; process.stdin.on('data', (d) => (raw += d)); @@ -63,7 +73,7 @@ process.stdin.on('end', () => { fileMode: (statSync(file).mode & 0o777).toString(8), urlMatches: config.mcpServers.app.url === call.mcp.url, runsInConfigDir: process.cwd() === dirname(file), - keySha: sha(key), + keyPrint: keyPrint(key), keyOnArgv: process.argv.some((a) => a.includes(key)), keyInEnv: Object.values(process.env).some((v) => typeof v === 'string' && v.includes(key)), keyOnStdin: raw.includes(key),