diff --git a/.changeset/cli-usage-errors.md b/.changeset/cli-usage-errors.md new file mode 100644 index 0000000000..03359ba695 --- /dev/null +++ b/.changeset/cli-usage-errors.md @@ -0,0 +1,6 @@ +--- +'@fluentui-react-native/desktop-driver': patch +'@fluentui-react-native/storybook-desktop': patch +--- + +Report desktop-driver CLI usage errors in every reporter mode. Subcommand usage errors now reach stderr (plain) or a single `error` record (json and ndjson) and exit 1 instead of exiting silently. An invalid `DESKTOP_REPORTER` no longer lets `serve` start. NDJSON failures emit the error record as their terminal record, matching JSON. WebDriver errors map to matching CLI error codes, and `doctor` messages are clipped. diff --git a/.changeset/life-desktop-test-shutdown.md b/.changeset/life-desktop-test-shutdown.md new file mode 100644 index 0000000000..210bb2a4ef --- /dev/null +++ b/.changeset/life-desktop-test-shutdown.md @@ -0,0 +1,5 @@ +--- +'@fluentui-react-native/desktop-test': patch +--- + +Fix shutdown hangs and cancellation. An integration's run is no longer tracked in the scope it borrows, so a Storybook `serve` or story test exits after SIGINT instead of idling, and scope draining is bounded. Cancellation stops running stage and suite commands, including their POSIX process groups, reports them failed, and refuses late spawns. A second interrupt force-stops owned process trees, SIGHUP is handled, and handlers stay installed until cleanup completes; `handleDesktopInterrupts` exports that policy for integrations. On Windows, `taskkill` runs synchronously and only before the command's exit is recorded, so it cannot address a reused process ID. Close disposes services and resources in one reverse order even when an observer throws. Workload claims move to a per-user directory independent of TMPDIR and are replaced atomically; local sockets use short private paths. Generated WDIO configs read settings from a file, suites accept `suiteTimeoutMs`, and command arguments and environment reject NUL. diff --git a/.changeset/life-storybook-desktop-scope.md b/.changeset/life-storybook-desktop-scope.md new file mode 100644 index 0000000000..c5350856e0 --- /dev/null +++ b/.changeset/life-storybook-desktop-scope.md @@ -0,0 +1,5 @@ +--- +'@fluentui-react-native/storybook-desktop': patch +--- + +Leave an injected desktop-test service scope to its owner instead of closing it from `driver`, `server`, and the smoke lifecycle, which made a supervising `desktop-test serve` or story test wait on itself after a stop. Stage commands now receive the scope's cancellation signal. The CLI and the standalone Storybook channel keep their interrupt handlers until cleanup completes, handle SIGHUP, and force-stop owned commands on a second interrupt; an interrupted one-shot command reports `incomplete` instead of passed. diff --git a/.changeset/rep-desktop-test-verdicts.md b/.changeset/rep-desktop-test-verdicts.md new file mode 100644 index 0000000000..88a53d940a --- /dev/null +++ b/.changeset/rep-desktop-test-verdicts.md @@ -0,0 +1,5 @@ +--- +'@fluentui-react-native/desktop-test': patch +--- + +Fix verdicts that passed without coverage. A test takes the status of its decisive attempt, so a failed attempt followed only by skipped or unsupported retries stays failed. Selected worker work that reports no tests is `incomplete`, and the WDIO reporter warns when a spec ran no tests, with the jasmine and mocha framework loggers raised to `warn` so an import failure is visible. WDIO attempts keep their arrival order whatever numbers the runner reports. Console stalls, closed pipes, and queue overflow are warnings that no longer fail a passing run, and the final line comes from the terminal result. Incomplete runs list their unmet work and a JUnit message that names it, and result checks throw a typed `DesktopIncompleteError` that reporters record as the run's verdict instead of an infrastructure failure. Failures and late-registered secrets are sanitized once at finalization, and run-query errors use a closed code set without filesystem paths. NDJSON milestones no longer suppress a later failure after many skips, record IDs are not clipped, a projection failure still emits a summary record, `run.json` snapshots are throttled, and producers can no longer send `run-finished`. diff --git a/packages/agentic/desktop-driver/references/getting-started.md b/packages/agentic/desktop-driver/references/getting-started.md index e08a928dfb..9eed2b3a9e 100644 --- a/packages/agentic/desktop-driver/references/getting-started.md +++ b/packages/agentic/desktop-driver/references/getting-started.md @@ -111,7 +111,10 @@ desktop-driver stories list --platform windows --url http://127.0.0.1:7007 --tar `--reporter auto|plain|github-actions|json|ndjson` is also configurable with `DESKTOP_REPORTER`; `--json` aliases `--reporter json`. In `auto`, existing indented JSON is retained. One-shot commands write one result, while `serve` -emits one readiness result and then waits for shutdown. +emits one readiness result and then waits for shutdown. A usage or +configuration error in `json` or `ndjson` writes one `error` record +(`{"schemaVersion":1,"type":"error","error":{"code","message"}}`) to stdout and +exits 1; `plain` writes the message to stderr. ## Resolve and diagnose diff --git a/packages/agentic/desktop-driver/references/remaining-work.md b/packages/agentic/desktop-driver/references/remaining-work.md index e5e39c63c8..cf88cc7d10 100644 --- a/packages/agentic/desktop-driver/references/remaining-work.md +++ b/packages/agentic/desktop-driver/references/remaining-work.md @@ -425,12 +425,12 @@ alias for `--reporter json`. Resolve the mode from the flag, then `DESKTOP_REPORTER`, then configuration, then `auto`. Do not try to detect whether the caller is an agent. -| Mode | stdout | stderr | -| ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- | -| `auto` (default) | `github-actions` when `GITHUB_ACTIONS=true`; otherwise `plain`. A future interactive renderer may be selected only for a TTY. | Current diagnostics | -| `plain`, `github-actions` | Existing human renderers. `reporting.console` remains a compatible configuration source. | Current diagnostics | -| `json` | Exactly one compact document at exit. Long-lived commands (`serve`, Storybook `driver` and `server`) print one readiness document instead and nothing afterward. | Empty, except for failures before mode resolution or loss of stdout | -| `ndjson` | Compact LF-terminated records ending with exactly one terminal record, identical to the `json` document. One-shot commands emit one record. | Same as `json` | +| Mode | stdout | stderr | +| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- | +| `auto` (default) | `github-actions` when `GITHUB_ACTIONS=true`; otherwise `plain`. A future interactive renderer may be selected only for a TTY. | Current diagnostics | +| `plain`, `github-actions` | Existing human renderers. `reporting.console` remains a compatible configuration source. | Current diagnostics | +| `json` | Exactly one compact document at exit. Long-lived commands (`serve`, Storybook `driver` and `server`) print one readiness document instead and nothing afterward. | Empty, except for failures before mode resolution or loss of stdout | +| `ndjson` | Compact LF-terminated records. A run ends with exactly one `terminal` record, identical to the `json` document. A failure before a run starts emits only its `error` record, identical to `json`. | Same as `json` | `--events ` applies to `ndjson`; the default is `milestones`: @@ -714,7 +714,7 @@ exit-path gaps. **Complete when:** every reporter mode is tested on every exit path: pass, test failure, infrastructure failure, usage or configuration error, interruption, and closed stdout. Structured stdout must parse as exactly one -document or as valid NDJSON ending in one terminal record, and structured +document or as valid NDJSON ending in one terminal or error record, and structured stderr must be empty on normal paths. Documents and default streams stay within limits for synthetic runs with at least 10,000 tests, 1,000 failures, and 1 MiB messages. Secrets remain redacted, and delegated runs render once. diff --git a/packages/agentic/desktop-driver/src/cli/DesktopDriverCli.test.ts b/packages/agentic/desktop-driver/src/cli/DesktopDriverCli.test.ts index 2e7aa88299..c6c8b4176c 100644 --- a/packages/agentic/desktop-driver/src/cli/DesktopDriverCli.test.ts +++ b/packages/agentic/desktop-driver/src/cli/DesktopDriverCli.test.ts @@ -7,7 +7,7 @@ import path from 'node:path'; import { NativeDriverError } from '../native/NativeDriverError'; import type { NativeDriverArtifact } from '../native/types'; import type { DesktopStoryManifest } from '../storybook.js'; -import { createDesktopDriverCommand } from './createDesktopDriverCommand'; +import { createDesktopDriverCommand, runDesktopDriverCli } from './createDesktopDriverCommand'; jest.setTimeout(30_000); @@ -134,6 +134,84 @@ describe('desktop-driver CLI', () => { } }); + test('reports subcommand usage errors on stderr in plain mode', async () => { + const result = await runInProcessCli(['stories', 'list']); + + expect(result.stdout).toBe(''); + expect(result.stderr).toContain("required option '--url ' not specified"); + expect(result.exitCode).toBe(1); + }); + + test('reports subcommand usage errors as one structured record in JSON and NDJSON modes', async () => { + const json = await runInProcessCli(['--reporter', 'json', 'stories', 'list']); + const ndjson = await runInProcessCli(['--reporter', 'ndjson', 'stories', 'list']); + + for (const result of [json, ndjson]) { + expect(result.stderr).toBe(''); + expect(result.exitCode).toBe(1); + expect(JSON.parse(result.stdout)).toMatchObject({ + schemaVersion: 1, + type: 'error', + error: { code: 'usage', message: expect.stringContaining("required option '--url ' not specified") }, + }); + } + expect(ndjson.stdout).toBe(json.stdout); + }); + + test('reports bad choices and invalid option values as usage errors', async () => { + const choice = await runInProcessCli([ + '--reporter', + 'ndjson', + 'stories', + 'list', + '--platform', + 'linux', + '--url', + 'https://127.0.0.1:1', + '--target', + 'target', + ]); + const port = await runInProcessCli(['--reporter', 'json', 'serve', '--port', 'abc']); + + expect(JSON.parse(choice.stdout)).toMatchObject({ + type: 'error', + error: { code: 'usage', message: expect.stringContaining('Allowed choices are macos, windows') }, + }); + expect(JSON.parse(port.stdout)).toMatchObject({ + type: 'error', + error: { code: 'usage', message: expect.stringContaining('Port must be an integer between 0 and 65535') }, + }); + }); + + test('does not start serve when DESKTOP_REPORTER is invalid', async () => { + const result = await runInProcessCli(['serve', '--port', '0'], 'xml'); + + expect(result.stdout).toBe(''); + expect(result.stderr).toContain('Reporter must be one of auto, plain, github-actions, json, ndjson. Received "xml".'); + expect(result.exitCode).toBe(1); + }); + + test('clips long doctor failure messages', async () => { + const output: string[] = []; + const program = createDesktopDriverCommand({ + resolveDriver: jest.fn(async () => { + throw new Error('native helper failure '.repeat(200)); + }), + stdout: { + write(value) { + output.push(String(value)); + return true; + }, + }, + }); + + await program.parseAsync(['node', 'test', 'doctor', '--platform', 'macos']); + + const message = JSON.parse(output.join('')).error.message as string; + expect(Buffer.byteLength(message)).toBeLessThanOrEqual(1024); + expect(message.endsWith('…')).toBe(true); + }); + test('does not crash when the consumer closes stdout before a structured error is written', async () => { const response = await new Promise<{ code: number | null; stderr: string }>((resolve, reject) => { const child = spawnCli(['--reporter', 'json', 'unknown']); @@ -417,6 +495,32 @@ describe('desktop-driver CLI', () => { } }); +async function runInProcessCli(argv: readonly string[], reporterEnvironment?: string) { + const previousReporter = process.env.DESKTOP_REPORTER; + const stdout: string[] = []; + const stderr: string[] = []; + const writeOut = jest.spyOn(process.stdout, 'write').mockImplementation((chunk: string | Uint8Array) => { + stdout.push(String(chunk)); + return true; + }); + const writeErr = jest.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array) => { + stderr.push(String(chunk)); + return true; + }); + try { + if (reporterEnvironment === undefined) delete process.env.DESKTOP_REPORTER; + else process.env.DESKTOP_REPORTER = reporterEnvironment; + process.exitCode = undefined; + await runDesktopDriverCli(['node', 'desktop-driver', ...argv]); + return { stdout: stdout.join(''), stderr: stderr.join(''), exitCode: process.exitCode }; + } finally { + writeOut.mockRestore(); + writeErr.mockRestore(); + if (previousReporter === undefined) delete process.env.DESKTOP_REPORTER; + else process.env.DESKTOP_REPORTER = previousReporter; + } +} + function spawnCli(args: readonly string[]) { return spawn(process.execPath, [path.resolve(__dirname, '../../config/cli.cjs'), ...args], { stdio: ['ignore', 'pipe', 'pipe'], diff --git a/packages/agentic/desktop-driver/src/cli/createDesktopDriverCommand.ts b/packages/agentic/desktop-driver/src/cli/createDesktopDriverCommand.ts index f3a852273d..34c2ed305c 100644 --- a/packages/agentic/desktop-driver/src/cli/createDesktopDriverCommand.ts +++ b/packages/agentic/desktop-driver/src/cli/createDesktopDriverCommand.ts @@ -1,13 +1,15 @@ import { spawn } from 'node:child_process'; import fs from 'node:fs'; -import { Command, Option } from 'commander'; +import { Command, InvalidArgumentError, Option } from 'commander'; import { connectDesktopAgent } from '../agent/DesktopAgent.js'; import { validateDesktopStoryTests } from '../authoring/storyTests.js'; import { FakeDesktopHost } from '../hosts/fake/FakeDesktopHost.js'; import { buildNativeDesktopDriver, resolveNativeDesktopDriver } from '../native/nativeDriver.js'; import { + configureDesktopCliCommandTree, + describeDesktopCliError, desktopReporterModes, formatDesktopReporterError, formatDesktopReporterValue, @@ -66,18 +68,22 @@ export function createDesktopDriverCommand(options: CreateDesktopDriverCommandOp .name('desktop-driver') .description('Drive React Native desktop applications through W3C WebDriver without Appium.') .addOption(new Option('--reporter ', 'output format').choices([...desktopReporterModes])) - .option('--json', 'alias for --reporter json') - .configureOutput({ - writeErr: (value) => stderr.write(value), - writeOut: (value) => stdout.write(value), - }); + .option('--json', 'alias for --reporter json'); + let resolvedReporterMode: ResolvedDesktopReporterMode | undefined; const reporterMode = (): ResolvedDesktopReporterMode => { - const flags = program.opts<{ reporter?: string; json?: boolean }>(); - return resolveDesktopReporterMode({ - flag: flags.json ? 'json' : flags.reporter, - environment: process.env.DESKTOP_REPORTER, - }); + if (!resolvedReporterMode) { + const flags = program.opts<{ reporter?: string; json?: boolean }>(); + resolvedReporterMode = resolveDesktopReporterMode({ + flag: flags.json ? 'json' : flags.reporter, + environment: process.env.DESKTOP_REPORTER, + }); + } + return resolvedReporterMode; }; + // Resolving before any action keeps an invalid DESKTOP_REPORTER from running a command such as serve. + program.hook('preAction', () => { + reporterMode(); + }); const writeValue = (value: unknown) => stdout.write(formatDesktopReporterValue(reporterMode(), value)); addNativeBuildCommand(program, buildDriver, writeValue); @@ -117,8 +123,8 @@ export function createDesktopDriverCommand(options: CreateDesktopDriverCommandOp }, ], }); - writeValue({ server: server.url, targetId: flags.target }); try { + writeValue({ server: server.url, targetId: flags.target }); await waitForSignal(); } finally { await server.close(); @@ -234,6 +240,10 @@ export function createDesktopDriverCommand(options: CreateDesktopDriverCommandOp }), ); + configureDesktopCliCommandTree(program, { + writeErr: (value) => stderr.write(value), + writeOut: (value) => stdout.write(value), + }); return program; } @@ -306,7 +316,7 @@ function addNativeDoctorCommand( writeValue({ error: { code: error instanceof Error && 'code' in error ? String(error.code) : 'unknown', - message: error instanceof Error ? error.message : String(error), + message: describeDesktopCliError(error), }, ready: false, }); @@ -330,7 +340,7 @@ export async function runDesktopDriverCli(argv: readonly string[] = process.argv reporterMode = 'plain'; } } - const program = createDesktopDriverCommand({ stderr: { write: (text) => ((diagnostics += text), true) } }).exitOverride(); + const program = createDesktopDriverCommand({ stderr: { write: (text) => ((diagnostics += text), true) } }); try { await program.parseAsync([...argv]); } catch (error) { @@ -403,7 +413,7 @@ function toResolveOptions(flags: NativeResolveFlags): NativeDriverResolveOptions function parseChoice(value: string, choices: readonly T[], name: string): T { if (!choices.includes(value as T)) { - throw new TypeError(`${name} must be one of ${choices.join(', ')}. Received "${value}".`); + throw new InvalidArgumentError(`${name} must be one of ${choices.join(', ')}. Received "${value}".`); } return value as T; } @@ -457,7 +467,7 @@ function waitForSignal(): Promise { function parsePort(value: string): number { const port = Number(value); if (!Number.isInteger(port) || port < 0 || port > 65_535) { - throw new TypeError(`Port must be an integer between 0 and 65535. Received "${value}".`); + throw new InvalidArgumentError(`Port must be an integer between 0 and 65535. Received "${value}".`); } return port; @@ -465,7 +475,7 @@ function parsePort(value: string): number { function requireOption(value: string | undefined, name: string): string { if (!value) { - throw new TypeError(`${name} requires a non-empty value.`); + throw new InvalidArgumentError(`${name} requires a non-empty value.`); } return value; } @@ -473,7 +483,7 @@ function requireOption(value: string | undefined, name: string): string { function parseNonNegativeInteger(value: string): number { const parsed = Number(value); if (!Number.isInteger(parsed) || parsed < 0) { - throw new TypeError(`Expected a non-negative integer. Received "${value}".`); + throw new InvalidArgumentError(`Expected a non-negative integer. Received "${value}".`); } return parsed; } @@ -481,7 +491,7 @@ function parseNonNegativeInteger(value: string): number { function parsePositiveInteger(value: string): number { const parsed = Number(value); if (!Number.isInteger(parsed) || parsed < 1) { - throw new TypeError(`Expected a positive integer. Received "${value}".`); + throw new InvalidArgumentError(`Expected a positive integer. Received "${value}".`); } return parsed; } diff --git a/packages/agentic/desktop-driver/src/cli/reporter.test.ts b/packages/agentic/desktop-driver/src/cli/reporter.test.ts index eade754391..002ae4629b 100644 --- a/packages/agentic/desktop-driver/src/cli/reporter.test.ts +++ b/packages/agentic/desktop-driver/src/cli/reporter.test.ts @@ -1,6 +1,10 @@ import { PassThrough } from 'node:stream'; +import { Command } from 'commander'; + import { + configureDesktopCliCommandTree, + describeDesktopCliError, formatDesktopReporterError, formatDesktopReporterValue, projectDesktopCliError, @@ -78,4 +82,49 @@ describe('desktop reporter modes', () => { error: { code: 'internal', message: 'invalid' }, }); }); + + test('reports NDJSON errors as the same record as JSON, without a terminal wrapper', () => { + const error = Object.assign(new Error('bad input'), { code: 'commander.invalidArgument' }); + expect(formatDesktopReporterError('ndjson', error)).toBe(formatDesktopReporterError('json', error)); + expect(JSON.parse(formatDesktopReporterError('ndjson', error))).toEqual({ + schemaVersion: 1, + type: 'error', + error: { code: 'usage', message: 'bad input' }, + }); + }); + + test.each([ + { label: 'invalid argument', fields: { code: 'invalid argument' }, kind: 'usage' }, + { label: 'invalid selector', fields: { code: 'invalid selector' }, kind: 'usage' }, + { label: 'unsupported operation', fields: { code: 'unsupported operation' }, kind: 'unsupported' }, + { label: 'webdriver client name', fields: { name: 'unsupported operation' }, kind: 'unsupported' }, + { label: 'target-busy session', fields: { code: 'session not created', data: { reason: 'target-busy' } }, kind: 'busy' }, + { label: 'session not created', fields: { code: 'session not created' }, kind: 'internal' }, + { label: 'no such element', fields: { code: 'no such element' }, kind: 'internal' }, + ])('maps WebDriver $label errors to $kind', ({ fields, kind }) => { + expect(projectDesktopCliError(Object.assign(new Error('x'), fields)).error.code).toBe(kind); + }); + + test('routes usage errors from nested subcommands to the reporter', async () => { + const root = new Command('root'); + root + .command('group') + .command('leaf') + .requiredOption('--url ') + .action(() => undefined); + const stderr: string[] = []; + configureDesktopCliCommandTree(root, { writeErr: (text) => stderr.push(text), writeOut: () => undefined }); + + const error: unknown = await root.parseAsync(['group', 'leaf'], { from: 'user' }).catch((caught: unknown) => caught); + + expect(error).toMatchObject({ code: 'commander.missingMandatoryOptionValue', exitCode: 1 }); + expect(projectDesktopCliError(error).error.code).toBe('usage'); + expect(stderr.join('')).toContain("required option '--url ' not specified"); + }); + + test('clips messages to the byte limit at a character boundary', () => { + const message = describeDesktopCliError(new Error('é'.repeat(2_000))); + expect(Buffer.byteLength(message)).toBeLessThanOrEqual(1024); + expect(message.endsWith('…')).toBe(true); + }); }); diff --git a/packages/agentic/desktop-driver/src/cli/reporter.ts b/packages/agentic/desktop-driver/src/cli/reporter.ts index b885a9ede7..ce151abdfa 100644 --- a/packages/agentic/desktop-driver/src/cli/reporter.ts +++ b/packages/agentic/desktop-driver/src/cli/reporter.ts @@ -1,3 +1,5 @@ +import type { Command } from 'commander'; + export const desktopReporterModes = ['auto', 'plain', 'github-actions', 'json', 'ndjson'] as const; export const desktopEventLevels = ['milestones', 'progress', 'all'] as const; @@ -18,6 +20,20 @@ export type DesktopReporterModeOptions = { githubActions?: boolean; }; +export type DesktopCliOutput = { + writeErr: (text: string) => void; + writeOut: (text: string) => void; +}; + +const maximumDesktopCliMessageBytes = 1024; + +// W3C WebDriver error names that describe a caller's input or an unsupported capability rather than a server fault. +const webDriverErrorKinds: Partial> = { + 'invalid argument': 'usage', + 'invalid selector': 'usage', + 'unsupported operation': 'unsupported', +}; + // CI templates and Windows shells commonly export an empty variable for an unset value. function nonEmpty(value: string | undefined): string | undefined { return value?.trim() ? value : undefined; @@ -88,19 +104,46 @@ export function tolerateClosedDesktopStdout(stream: NodeJS.WriteStream = process }); } +/** + * Routes commander usage errors and help to the reporter instead of the process. Commander copies these settings only + * into commands created after they are set, so they are applied to every command already in the tree. + */ +export function configureDesktopCliCommandTree(command: Command, output: DesktopCliOutput): void { + command.exitOverride().configureOutput(output); + for (const child of command.commands) configureDesktopCliCommandTree(child, output); +} + +export function describeDesktopCliError(error: unknown): string { + return clipReporterText(error instanceof Error ? error.message : String(error), maximumDesktopCliMessageBytes); +} + export function projectDesktopCliError(error: unknown): DesktopCliError { - const candidate = error && typeof error === 'object' && 'code' in error ? String(error.code) : undefined; - const code: DesktopCliErrorCode = candidate?.startsWith('commander.') - ? 'usage' - : isDesktopCliErrorCode(candidate) - ? candidate - : 'internal'; - const message = error instanceof Error ? error.message : String(error); - return { schemaVersion: 1, type: 'error', error: { code, message: clipReporterText(message, 1024) } }; + return { + schemaVersion: 1, + type: 'error', + error: { code: desktopCliErrorCode(error), message: describeDesktopCliError(error) }, + }; } +/** Compact NDJSON carries the error record as its single terminal record, so it matches the JSON document. */ export function formatDesktopReporterError(mode: ResolvedDesktopReporterMode, error: unknown): string { - return formatDesktopReporterValue(mode, projectDesktopCliError(error)); + return formatDesktopReporterValue(mode === 'ndjson' ? 'json' : mode, projectDesktopCliError(error)); +} + +function desktopCliErrorCode(error: unknown): DesktopCliErrorCode { + const record = error && typeof error === 'object' ? (error as { code?: unknown; name?: unknown; data?: unknown }) : undefined; + const code = typeof record?.code === 'string' ? record.code : undefined; + if (code?.startsWith('commander.')) return 'usage'; + if (isDesktopCliErrorCode(code)) return code; + + // WebDriver errors carry the W3C code in `code` (DD) or `name` (webdriver client errors). + const webDriverCode = code ?? (typeof record?.name === 'string' ? record.name : undefined); + if (webDriverCode === 'session not created' && hasTargetBusyReason(record?.data)) return 'busy'; + return (webDriverCode === undefined ? undefined : webDriverErrorKinds[webDriverCode]) ?? 'internal'; +} + +function hasTargetBusyReason(data: unknown): boolean { + return Boolean(data && typeof data === 'object' && 'reason' in data && data.reason === 'target-busy'); } function isDesktopCliErrorCode(value: string | undefined): value is DesktopCliErrorCode { diff --git a/packages/agentic/desktop-test/fixtures/neutral-boundary/standalone-consumer.mjs b/packages/agentic/desktop-test/fixtures/neutral-boundary/standalone-consumer.mjs index 4c210d384d..c2015a7961 100644 --- a/packages/agentic/desktop-test/fixtures/neutral-boundary/standalone-consumer.mjs +++ b/packages/agentic/desktop-test/fixtures/neutral-boundary/standalone-consumer.mjs @@ -8,6 +8,23 @@ import { startDesktopService, startNativeDesktopService } from '@fluentui-react- const catalog = describeOperations({ accessibilityActions: false, boundedObservation: false }); +// Every public library entry loads on its own, without Storybook or a running WDIO session. +const entries = { + '.': await import('@fluentui-react-native/desktop-test'), + './config': await import('@fluentui-react-native/desktop-test/config'), + './reporting': await import('@fluentui-react-native/desktop-test/reporting'), + './reporting/run-queries': await import('@fluentui-react-native/desktop-test/reporting/run-queries'), + './commands': await import('@fluentui-react-native/desktop-test/commands'), + './driver-service': await import('@fluentui-react-native/desktop-test/driver-service'), + './process-control': await import('@fluentui-react-native/desktop-test/process-control'), + './resources': await import('@fluentui-react-native/desktop-test/resources'), + './ownership': await import('@fluentui-react-native/desktop-test/ownership'), + './jasmine': await import('@fluentui-react-native/desktop-test/jasmine'), + './wdio': await import('@fluentui-react-native/desktop-test/wdio'), +}; +const empty = Object.entries(entries).filter(([, module]) => Object.keys(module).length === 0); +if (empty.length) throw new Error(`Public desktop-test entries exported nothing: ${empty.map(([entry]) => entry).join(', ')}`); + if ( typeof createDesktopDriverClient !== 'function' || typeof DesktopAuthority !== 'function' || diff --git a/packages/agentic/desktop-test/references/commands.md b/packages/agentic/desktop-test/references/commands.md index 0dcec90136..c25b3a4f9e 100644 --- a/packages/agentic/desktop-test/references/commands.md +++ b/packages/agentic/desktop-test/references/commands.md @@ -44,6 +44,13 @@ target served by an imported integration, `run` receives the output format, writes its own readiness document, and calls `onReady` once it has. Clients create sessions explicitly. +Stop any command with Ctrl+C, SIGTERM, or SIGHUP. The first interrupt stops the +running stage or suite and every owned service, and the command exits once +cleanup completes, including a `serve` whose target is served by an imported +integration. A second interrupt force-kills owned process trees; a third exits +at once and may leave the workload claim behind. See +[lifecycle](lifecycle.md#lifecycle-and-ownership). + `serve --agent` makes the service require a grant on every route, issues one grant for the target, and publishes it with the service URL and the helper's description of this process as a private connection descriptor, which the diff --git a/packages/agentic/desktop-test/references/configuration.md b/packages/agentic/desktop-test/references/configuration.md index 4a780442a7..ea2c40165c 100644 --- a/packages/agentic/desktop-test/references/configuration.md +++ b/packages/agentic/desktop-test/references/configuration.md @@ -22,14 +22,18 @@ identity. External ownership permits attach only and preserves the app. `commands` overrides individual `prep`, `bundle`, `build`, or `run` stages. Each command has an executable and argument array; no shell-like string parsing -is performed. Required services are declared in config and a service exit is a +is performed. Arguments and environment entries must not contain NUL, which no +process can receive. Required services are declared in config and a service exit is a failure even after readiness. ## Add a suite workspace A separate suite references the app's canonical target using `use: { config, target }`, and declares grouped spec globs, framework, hooks, retry policy, and -instance limits. Install WDIO CLI, local runner, and the selected framework in +instance limits. `timeoutMs` bounds each test. `suiteTimeoutMs` bounds the whole +WDIO run, so a wedged launcher fails instead of hanging; by default it is +`timeoutMs` (60 seconds when unset) times 100, times the number of spec groups +and attempts, and at least five minutes. Install WDIO CLI, local runner, and the selected framework in the suite workspace; the harness does not install test frameworks into app workspaces. diff --git a/packages/agentic/desktop-test/references/current-status.md b/packages/agentic/desktop-test/references/current-status.md index 785725a73a..5850b32c1d 100644 --- a/packages/agentic/desktop-test/references/current-status.md +++ b/packages/agentic/desktop-test/references/current-status.md @@ -64,14 +64,14 @@ did not retain a portable raw-report locator; do not imply otherwise. ## Full-submission checklist -| Area | Remaining evidence/work | -| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| macOS | Live failed-cleanup and service-restart cases; Paper separately; supervisor death with Metro/channel/helper survivors; exact attached-app preservation. | -| Windows / Win32 | Same-branch MCP restart/revocation/dispose/args tests; physical console Ctrl+C and the idle `serve --agent` after interrupt; interactive input/capture rerun; naming launched (especially uncontained packaged) apps in the claim. | -| Storybook | Authenticated mutation ingress, context fencing/reconnect, consistent helper build policy, manifest provenance and separate nonces. | -| Recovery | Durable uncertainty is wired in both paths; live restart qualification, ledger and exact-resource reconciliation still precede automatic reclaim. | -| Concurrency | No physical-input suites in parallel on one interactive desktop. Separate machines may qualify independent endpoints. | -| Reliability | Establish the required repeated-run window rather than interpreting aggregate past successes as a pass. | +| Area | Remaining evidence/work | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| macOS | Live failed-cleanup and service-restart cases; Paper separately; supervisor death with Metro/channel/helper survivors; exact attached-app preservation. | +| Windows / Win32 | Same-branch MCP restart/revocation/dispose/args tests; physical console Ctrl+C requalification of `serve --agent` after the idle-CLI fix; interactive input/capture rerun; naming launched (especially uncontained packaged) apps in the claim. | +| Storybook | Authenticated mutation ingress, context fencing/reconnect, consistent helper build policy, manifest provenance and separate nonces. | +| Recovery | Durable uncertainty is wired in both paths; live restart qualification, ledger and exact-resource reconciliation still precede automatic reclaim. The workload claim records a Node.js-clock start time, so it needs a helper-described start time before claim inspection can prove a live owner or a reused PID. | +| Concurrency | No physical-input suites in parallel on one interactive desktop. Separate machines may qualify independent endpoints. | +| Reliability | Establish the required repeated-run window rather than interpreting aggregate past successes as a pass. | Tracking: live macOS, Windows, and Win32 requalification [#4441](https://github.com/microsoft/fluentui-react-native/issues/4441); @@ -691,6 +691,15 @@ process stayed idle without exiting. Each time, the exact owner was ended only after that proof, and then its stale claim was removed. The physical console Ctrl+C case is otherwise out of scope for this pass. +The idle CLI had one cause on every platform: the CLI tracked the Storybook +integration's run in the service scope it lent to that run, and the run closed +the scope, so closing waited on itself. On macOS, a Storybook-integration +`serve` was still running 90 seconds after SIGINT, with its claim `active`. +Integrations now leave the borrowed scope to the CLI, the CLI waits for the run +outside the scope with a bound, and scope draining is bounded. With that fix the +same macOS `serve` exits promptly after SIGINT and leaves no owned process or +claim. A physical console Ctrl+C on Windows and Win32 still needs requalification. + **Reconciled claims.** Every stale or fenced claim in this pass was removed by hand only after its owner was proved dead and its resources were proved settled, or after the fenced launch probe's survivor was ended. Nothing @@ -741,4 +750,5 @@ only after identity verification. Both deliberately stopped Win32 app runs, The standalone drivers were stopped by a session-local injected SIGTERM, not physical Ctrl+C. Both owned app/service processes exited, the recorded ports were free, and the per-user claim and default descriptor were absent. -The separate `serve --agent` interrupt limitation above remains open. +The separate `serve --agent` interrupt limitation above is fixed but not yet +requalified on Windows. diff --git a/packages/agentic/desktop-test/references/lifecycle.md b/packages/agentic/desktop-test/references/lifecycle.md index 109a7a7ed7..bdbcfcd28e 100644 --- a/packages/agentic/desktop-test/references/lifecycle.md +++ b/packages/agentic/desktop-test/references/lifecycle.md @@ -17,7 +17,10 @@ reverse order; borrowed resources are recorded but never disposed. A launched application's lifetime is currently session-owned; attached applications have an external owner. Storybook still takes over the CLI lifecycle, with channel and embedded driver listeners in a child process. This exception is -transitional, not a fully migrated resource graph. The intended boundary is one +transitional, not a fully migrated resource graph. An integration's `run` borrows +the supervisor's service scope: it observes the scope signal and must never close +the scope, which the CLI closes after the run settles. Once a stop is requested, +the CLI waits at most 60 seconds for the run before it closes the scope anyway. The intended boundary is one neutral `desktop-test` workload host with optional Storybook contributions, not a driver or MCP orchestrator. @@ -39,30 +42,65 @@ Only `service-ready` is accepted; target acquisition, application readiness, and context readiness require separate verification. Cleanup drains worker reporting and stdio, closes sessions, releases owned -input, and stops exact owned process trees in reverse service order. An -externally owned app is preserved. +input, and stops exact owned process trees. Services and resources are disposed +together in reverse acquisition order. An externally owned app is preserved. + +Cancellation reaches running commands. Stage commands and WDIO suites receive +the scope signal; when it aborts, the command's process tree is stopped and the +command reports failed with "Stopped before it finished.", never passed. Services +are stopped by close, in the same reverse order. On macOS and Linux each command +leads its own process group, and stopping it signals the whole group, including +descendants whose parent already exited: SIGTERM, then SIGKILL if the group +outlives the shutdown timeout. A process that leaves the group, such as an app +started through LaunchServices, needs its own owned resource. On Windows, +`taskkill /T /F` stops the tree only while the command's exit is unrecorded, +and runs synchronously: Windows reuses a process ID only after every handle to +the process closes, and Node closes its handle while processing the exit on the +same event loop, so the ID cannot name another process. A failed `taskkill` is +judged by the recorded exit, never by probing the ID. This is not containment: +`/T` follows parent process IDs, so a descendant whose parent already exited +escapes until Job Object containment exists. A command never starts after its +signal has aborted, and a spawn that throws synchronously leaves no tracked +process behind. + +Interrupts stay handled until cleanup completes. The first SIGINT, SIGTERM, or +SIGHUP requests a graceful stop; the second force-kills owned process trees +while cleanup continues; the third exits immediately with status 128 plus the +signal number and leaves the claim for inspection. An interrupted one-shot run +reports `incomplete`. Commands keep their arguments as separate values. Windows executable lookup and command-shim escaping use `cross-spawn`, without Node's deprecated `spawn` argument array plus `shell: true` combination. Interactive runs acquire a per-user `interactive.lock`; concurrent workloads -fail closed. If a crash leaves a lock, inspect its recorded owner and verify +fail closed. The claim lives in `~/Library/Caches/com.microsoft.fluentui-react-native.desktop-test/workload` +on macOS, `%LOCALAPPDATA%\Microsoft\FluentUIReactNative\desktop-test\workload` +on Windows, and `~/.cache/fluentui-react-native/desktop-test/workload` elsewhere, +independent of `TMPDIR`, so every checkout contends for the same claim. Claim +records are replaced atomically. If a crash leaves a lock, inspect its recorded owner and verify that process and its resources have stopped before removing that exact lock. There is no automatic stale-lock recovery. The supervisor registers each owned resource before acquisition starts. If cancellation wins while acquisition is in flight, shutdown waits for that work; a resource that arrives late is disposed before a successful shutdown settles. -Guarded work is also drained before close completes. A failed or timed-out -disposal remains unsettled and retains the claim. +Guarded work is also drained before close completes, for at most the scope's +`drainTimeoutMs` (60 seconds by default); work that is still running then is +reported as a cleanup failure, and disposal proceeds. A resource observer that +throws does not stop disposal; its failure is reported after cleanup. A failed or +timed-out disposal remains unsettled and retains the claim. The versioned workload record stores a random owner ID, process ID and recorded start timestamp, run ID, registered target ID, physical target instance ID, and each resource's identity, generation, ownership, readiness, and disposal state. Unresolved effects fence the claim. Claim inspection distinguishes a live owner, an exited owner, an unknown probe, and a reused PID; inspection never reclaims a -claim, and uncertain or unsettled resources prevent release. Reporting-session +claim, and uncertain or unsettled resources prevent release. The recorded start +timestamp comes from the Node.js clock, not from the operating system, and +inspection compares it exactly, so a probe that reports the operating-system start +time classifies a live owner as a reused PID. Treat `pid-reused` as unproven until +the claim records a helper-described start time. Reporting-session inheritance is separate from explicit workload-claim delegation. Both native service paths use the real `driverService.ts` composition root diff --git a/packages/agentic/desktop-test/references/native-qualification.md b/packages/agentic/desktop-test/references/native-qualification.md index 9a26805786..7d0ae7dfbd 100644 --- a/packages/agentic/desktop-test/references/native-qualification.md +++ b/packages/agentic/desktop-test/references/native-qualification.md @@ -206,9 +206,10 @@ Windows: `--windows` is always Fabric and `--win32` is always Paper. instance's environment as described in `@fluentui-react-native/storybook-desktop/references/platforms/win32.md`; a stale bundle never reaches the channel. -- A console Ctrl+C to `desktop-test serve --agent` has left the CLI idle after - every resource settled, with its claim still `active` and `run.json` - `incomplete`. Record that, prove the owner and resources as in +- A console Ctrl+C to `desktop-test serve --agent` used to leave the CLI idle + after every resource settled, with its claim still `active` and `run.json` + `incomplete`. That is fixed; record whether the CLI now exits by itself. If + it stays idle, prove the owner and resources as in [cleanup](#cleanup-and-endpoint-limits), and end only that process. - If `storybook run --windows` fails while enabling Developer Mode, do not change that OS setting for qualification. An already installed, verified @@ -240,7 +241,7 @@ after cleanup evidence is recorded. On a macOS-only machine, record **Windows Fabric** and **Win32 Paper** as **Not run**, with no measurements inferred from macOS. On Windows, also confirm -that no `%TEMP%\furn-desktop-workload-*\interactive.lock` and no +that no `%LOCALAPPDATA%\Microsoft\FluentUIReactNative\desktop-test\workload\interactive.lock` and no `%LOCALAPPDATA%\com.microsoft.fluentui-react-native.desktop-driver\agent-connection.json` remain, that the channel, Metro, and WebDriver ports reported by the driver have no listeners, and that no app process from the run survives. Hosted physical-input diff --git a/packages/agentic/desktop-test/references/reporting.md b/packages/agentic/desktop-test/references/reporting.md index 47661bc059..efedfddfaf 100644 --- a/packages/agentic/desktop-test/references/reporting.md +++ b/packages/agentic/desktop-test/references/reporting.md @@ -33,6 +33,40 @@ capabilities, and unfinished work. A zero worker exit cannot override failed assertions or incomplete work. Optional suites can be skipped without being labelled passed. +## Verdicts + +A run is `failed`, `incomplete`, `skipped`, or `passed`, in that order of +precedence. Skipped capabilities, empty work, and incomplete work are never +passing coverage. + +- **Decisive attempt.** A test with retries takes the status of its decisive + attempt, not of its last one: a passing last attempt passes the test (it is + `flaky` if an earlier attempt failed); otherwise any failed attempt fails the + test; otherwise the last attempt's status stands. A failed attempt followed + only by `skipped`, `unsupported`, or `incomplete` retries therefore stays + `failed`, and JUnit and the summaries report that failure. +- **Attempt order.** Attempts keep the order in which they arrive. A producer's + attempt number only identifies the attempt (WDIO numbers a spec file's retries + `N+1, 1, 2, …`), so the collector never sorts by it. +- **Empty work.** Selected worker work (such as a WDIO spec group) that reports + no tests is `incomplete` with the reason "No tests were reported for this + work.", unless the producer declared exclusions for it. WDIO reports a spec + that defines no tests, or fails to import, as passed, so the reporter also + emits a warning diagnostic naming the specs. The generated WDIO configuration + keeps `logLevel: 'error'` but raises the jasmine and mocha framework loggers to + `warn`, so a spec import failure is visible. A run of kind `tests` that + executes no test is incomplete as well. +- **Incomplete runs.** An `incomplete` run lists each unmet item as a problem in + the JSON and NDJSON summaries (the work and its reason, tests that did not + finish, producers that never finished), and JUnit and the Actions summary + state the same reason. Result checks such as `assertSuccessful` throw a + `DesktopIncompleteError`, which a collector or delegated reporter recognizes by + name: `fail()` records it as the run's `incomplete` verdict, not as an + `infrastructure` failure. If nothing else in the run is incomplete, the same + error is a failure so the gap cannot disappear. +- **Collector-only events.** Only the collector emits `run-finished`; a producer + that sends it is rejected. + ## Console output A command, or a step such as a story test run, is an operation. When it ends the @@ -66,7 +100,22 @@ Without `--verbose` a passing, skipped, or unsupported test appears only in the final counts, so a run that executes nothing shows as `0 executed` instead of disappearing into a long list. Failed tests, retries, and diagnostics always print, and the root supervisor's stderr is always live. Every run finishes with a -`[desktop] PASSED: …` or `[desktop] FAILED: …` line carrying the counts. +`[desktop] : …` line carrying the counts, where the status is `PASSED`, +`FAILED`, `INCOMPLETE`, or `SKIPPED`, read from the final result. + +Console output is presentation, so its errors never change the verdict. A +consumer that closes the pipe (`EPIPE`), stalls past the flush timeout, or falls +more than 4 MiB behind (the queue counts structured NDJSON bytes too) stops +console output, and the reporter writes +`[desktop] WARNING: ; authoritative result: ` to stderr and a +warning diagnostic to the journal, after the final result is written. The result +and its retained reports are unaffected. + +Sensitive text is sanitized once, when the reports are written: the collector and +delegated reporters redact decoded failure values with one registry, never JSON +text, so a secret registered after its text was reported is still removed from +`run.json`, `junit.xml`, and the summaries. `events.jsonl` keeps the text it +journaled when it was reported. Output is untrusted text. The reporter neutralizes each line it prints that it did not build as a workflow command, so a command cannot start, stop, or forge @@ -97,14 +146,19 @@ event count, and the same summary as its final record. `--events` selects same level for delegated commands. Event selection is valid only with NDJSON. The summary is capped at 4 KiB and includes counts, up to 10 problems, omission counts, and links to the authoritative `run.json`, `events.jsonl`, JUnit, and -command logs. Structured output suppresses normal human stderr. If stdout -closes, the run continues to persist its result and records a reporting -diagnostic. - -SIGINT and SIGTERM cancel owned work; one-shot structured runs finish with an +command logs. Record IDs are never clipped: text fields shrink first, then the +record falls back to a minimal one marked `truncated: true`, so the summary +always arrives. Under `milestones`, `skipped` and `unsupported` test results +(including declared gaps) share a separate cap of 25 events; a `failed`, +`incomplete`, or `cancelled` result is never suppressed. Structured output suppresses normal +human stderr. If stdout closes, the run continues to persist its result and +reports a warning, as for [console output](#console-output). + +SIGINT, SIGTERM, and SIGHUP cancel owned work; one-shot runs finish with an `incomplete` summary instead of reporting cancellation as an infrastructure -failure. Long-lived readiness commands retain their single readiness document; -one that fails before readiness writes the summary instead. +failure, and a stage stopped before it finished is never reported as passed. +Long-lived readiness commands retain their single readiness document; one that +fails before readiness writes the summary instead. The supervisor owns rendering. Delegated producers inherit the selected mode and report into the parent journal; they do not create a second result @@ -181,8 +235,10 @@ MCP tool, or execution/recovery API. Its credentials are distinct from driver control grants. Connecting a reader only binds a credential; it never creates a workload, producer, controller session, or application. -The collector atomically replaces `run.json` with an active snapshot after each -batch of events (coalesced within a JavaScript turn). Active snapshots have +The collector atomically replaces `run.json` with an active snapshot after events +that can change the result (not output or progress diagnostics), at most every +250 ms. `flush()` and finalization always write the current state, so a reader +may see a snapshot up to 250 ms behind. Active snapshots have `terminal: false` and cannot have a passing status. Only successful final report persistence after producer/output accounting sets `terminal: true`. Terminal means collection finished, not that coverage passed. A crashed @@ -231,6 +287,13 @@ The host must register all relevant secrets (including credentials for any external integration); `addSecret` can update the read redactor. The collector redacts before persistence, and queries redact again before clipping text. +Query errors carry one code from a closed set: `unavailable`, `invalid-request`, +`stale-cursor`, `report-unreadable`, `report-changed`, `report-invalid`, or +`internal`. Messages are fixed text, so an error never exposes a path or the +reason a filesystem or parser call failed. Text returned from a retained report +replaces the run directory with `` and its parent directory with +``. + Readers submit only an opaque run ID, a declared operation, and its bounded parameters. `status` returns complete counts, terminal/coverage state, and the run's input `lane` and `excluded` counts when it has them. diff --git a/packages/agentic/desktop-test/src/adapters.ts b/packages/agentic/desktop-test/src/adapters.ts index f5251597b5..47d592bc4f 100644 --- a/packages/agentic/desktop-test/src/adapters.ts +++ b/packages/agentic/desktop-test/src/adapters.ts @@ -95,9 +95,10 @@ export async function resolveDesktopApplication( '-NoProfile', '-NonInteractive', '-Command', - `$ErrorActionPreference='Stop'; $p=@(Get-AppxPackage -Name '${name}'); if ($p.Count -ne 1) { throw 'Expected one registered app package; install the generated application first.' }; $p[0].PackageFamilyName`, + // The package name is passed as data in the environment, never spliced into the script. + "$ErrorActionPreference='Stop'; $p=@(Get-AppxPackage -Name $env:FURN_APPX_NAME); if ($p.Count -ne 1) { throw 'Expected one registered app package; install the generated application first.' }; $p[0].PackageFamilyName", ], - { timeout: 10_000, signal, maxBuffer: 64 * 1024, windowsHide: true }, + { timeout: 10_000, signal, maxBuffer: 64 * 1024, windowsHide: true, env: { ...process.env, FURN_APPX_NAME: name } }, ); const family = stdout.trim(); if (!/^[\w.-]+_[\w]+$/.test(family)) throw new Error('Could not verify the registered Windows package family.'); diff --git a/packages/agentic/desktop-test/src/cli.ts b/packages/agentic/desktop-test/src/cli.ts index cc2a039294..f675bcf789 100644 --- a/packages/agentic/desktop-test/src/cli.ts +++ b/packages/agentic/desktop-test/src/cli.ts @@ -1,3 +1,4 @@ +import { randomUUID } from 'node:crypto'; import { Command, Option } from 'commander'; import { buildNativeDesktopDriver } from '@fluentui-react-native/desktop-driver'; import { @@ -18,13 +19,18 @@ import type { DesktopAgentConnectionOptions } from './agentConnection.js'; import { loadDesktopConfig, resolveDesktopSpecs, selectDesktopRenderer } from './config.js'; import type { DesktopStage } from './config.js'; import { createDesktopInputLane, desktopInputEnvironment, requestedDesktopInputLane } from './reporting/lane.js'; +import { deadline } from './commands.js'; import { createDesktopReportSession } from './reporting/session.js'; import { DesktopStopRequested } from './services.js'; +import { handleDesktopInterrupts } from './signals.js'; import { DesktopSupervisor } from './supervisor.js'; import { assertDesktopSuiteSupportsInput, runDesktopWdioSuite } from './wdio/run.js'; type AgentFlags = { agent?: boolean; descriptor?: string; root?: string }; +// How long an integration may take to unwind after a stop request or scope failure before the scope is closed anyway. +const INTEGRATION_STOP_TIMEOUT_MS = 60_000; + export async function runDesktopTestCli(argv = process.argv): Promise { tolerateClosedDesktopStdout(); const program = new Command('desktop-test') @@ -193,9 +199,19 @@ export async function runDesktopTestCli(argv = process.argv): Promise { ...(input ? { input } : {}), }); session.onStop?.(() => owner.services.requestStop()); - const interrupt = () => owner.services.requestStop(); - process.once('SIGINT', interrupt); - process.once('SIGTERM', interrupt); + const releaseInterrupts = handleDesktopInterrupts({ + stop: () => owner.services.requestStop(), + terminate: (signal) => { + reporter.emit({ + kind: 'diagnostic', + payload: { + level: 'warning', + message: `Received ${signal} during shutdown; force-stopping owned commands. Interrupt again to exit without cleanup.`, + }, + }); + owner.runner.terminate(); + }, + }); let failure: unknown; try { if (action === 'build-driver') { @@ -212,7 +228,10 @@ export async function runDesktopTestCli(argv = process.argv): Promise { await owner.open(['test', 'serve', 'run'].includes(action)); await owner.startResources(); if (action === 'test' || action === 'serve') await owner.startConfiguredServices(); - await owner.services.guard( + owner.services.controller.signal.throwIfAborted(); + // The integration borrows this scope, and close() waits for the work the scope tracks, so the run is not tracked: + // it observes the scope signal and is awaited here, bounded, before the owner closes the scope. + const run = Promise.resolve( config.config.integration.run({ config, target, @@ -224,10 +243,34 @@ export async function runDesktopTestCli(argv = process.argv): Promise { ...(action === 'serve' ? { reporterMode, onReady: markReady } : {}), }), ); + const outcome = run.then( + () => undefined, + (error: unknown) => ({ error }), + ); + try { + await Promise.race([run, owner.services.waitForStop()]); + } catch (error) { + const late = await deadline(outcome, INTEGRATION_STOP_TIMEOUT_MS, 'Waiting for the integration to stop').catch( + (timeout: unknown) => ({ error: timeout }), + ); + const lateError = late?.error; + if ( + lateError === undefined || + lateError === error || + lateError instanceof DesktopStopRequested || + lateError === owner.services.controller.signal.reason + ) + throw error; + throw error instanceof DesktopStopRequested + ? lateError + : new AggregateError([error, lateError], 'The integration failed while stopping.'); + } } else if (action !== 'test' && action !== 'serve') { await owner.open(action === 'run'); for (const command of desktopStageCommands(target, action)) { - await owner.services.guard(owner.runner.run(owner.prepare(command, `${action} ${target.endpoint}`))); + await owner.services.guard( + owner.runner.run(owner.prepare(command, `${action} ${target.endpoint}`), { signal: owner.services.controller.signal }), + ); } } else { await owner.start(); @@ -250,10 +293,14 @@ export async function runDesktopTestCli(argv = process.argv): Promise { if (!(error instanceof DesktopStopRequested)) { failure = error; reporter.fail(error); + } else if (action !== 'serve') { + // An interrupted one-shot action did not finish: leave its scope open so the run reports incomplete, not passed. + reporter.emit( + { kind: 'scope-start', payload: { label: `${action} ${target.endpoint} (interrupted)`, category: 'phase' } }, + `interrupted-${randomUUID()}`, + ); } } finally { - process.off('SIGINT', interrupt); - process.off('SIGTERM', interrupt); try { await owner.close(); } catch (error) { @@ -261,7 +308,11 @@ export async function runDesktopTestCli(argv = process.argv): Promise { failure = failure === undefined ? error : new AggregateError([failure, error], 'Desktop execution and cleanup failed.'); } } - process.exitCode = await session.close(failure); + try { + process.exitCode = await session.close(failure); + } finally { + releaseInterrupts(); + } if (session.delegated && action !== 'serve' && (reporterMode === 'json' || reporterMode === 'ndjson')) { process.stdout.write( formatDesktopReporterValue(reporterMode, { runId: reporter.runId, delegated: true, exitCode: process.exitCode }), diff --git a/packages/agentic/desktop-test/src/commands.test.ts b/packages/agentic/desktop-test/src/commands.test.ts new file mode 100644 index 0000000000..c96e2afd4e --- /dev/null +++ b/packages/agentic/desktop-test/src/commands.test.ts @@ -0,0 +1,237 @@ +import { spawnSync } from 'node:child_process'; +import type { SpawnSyncReturns } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { deadline, NodeDesktopCommandRunner } from './commands.js'; +import type { PreparedDesktopCommand } from './commands.js'; +import { DesktopRunReporter } from './reporting/collector.js'; + +jest.mock('node:child_process', () => { + const actual = jest.requireActual('node:child_process'); + return { ...actual, spawnSync: jest.fn(actual.spawnSync) }; +}); + +const quiet = { write: () => true }; +jest.setTimeout(30_000); + +describe('desktop command cancellation and failure', () => { + let root: string; + let reporter: DesktopRunReporter; + let runner: NodeDesktopCommandRunner; + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-commands-')); + reporter = new DesktopRunReporter({ artifactsRoot: root, kind: 'lifecycle', output: quiet, errorOutput: quiet, summary: false }); + runner = new NodeDesktopCommandRunner({ reporter, shutdownTimeoutMs: 2000 }); + }); + afterEach(async () => { + runner.terminate(); + await runner.close().catch(() => undefined); + fs.rmSync(root, { force: true, recursive: true, maxRetries: 20, retryDelay: 100 }); + }); + + // Fixture code is constant; the marker path reaches it through process.argv (R-SCAN-002). + const node = (source: string, ...values: string[]): PreparedDesktopCommand => ({ + command: process.execPath, + args: ['-e', source, ...values], + cwd: root, + env: {}, + label: 'fixture', + }); + + function events(): { kind: string; payload: Record }[] { + return fs + .readFileSync(path.join(reporter.root, 'events.jsonl'), 'utf8') + .trim() + .split('\n') + .map((line) => JSON.parse(line)); + } + + test('an aborted run stops its process group, including a grandchild, and reports the command failed', async () => { + const marker = path.join(root, 'grandchild.pid'); + const controller = new AbortController(); + const started = Date.now(); + const run = runner.run( + node( + "const { spawn } = require('node:child_process');" + + "const grandchild = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' });" + + "require('node:fs').writeFileSync(process.argv[1], String(grandchild.pid));" + + 'setInterval(() => {}, 1000);', + marker, + ), + { signal: controller.signal }, + ); + await waitUntil(() => fs.existsSync(marker) && fs.readFileSync(marker, 'utf8').length > 0); + const grandchild = Number(fs.readFileSync(marker, 'utf8')); + controller.abort(new Error('Stage cancelled')); + await expect(run).rejects.toThrow('Stage cancelled'); + expect(Date.now() - started).toBeLessThan(10_000); + await waitUntil(() => !alive(grandchild)); + await reporter.finish(); + expect(events().filter((event) => event.kind === 'scope-end')).toEqual([ + expect.objectContaining({ payload: expect.objectContaining({ status: 'failed', summary: 'Stopped before it finished.' }) }), + ]); + }); + + test('an aborted signal refuses to spawn', async () => { + const marker = path.join(root, 'spawned'); + const controller = new AbortController(); + controller.abort(new Error('Already cancelled')); + const command = node("require('node:fs').writeFileSync(process.argv[1], 'spawned');", marker); + expect(() => runner.start(command, { signal: controller.signal })).toThrow('Already cancelled'); + await expect(runner.run(command, { signal: controller.signal })).rejects.toThrow('Already cancelled'); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(fs.existsSync(marker)).toBe(false); + }); + + test('rejects NUL bytes before spawning anything', () => { + expect(() => runner.start(node('', 'bad\0argument'))).toThrow('NUL'); + expect(() => runner.start({ ...node(''), env: { VALUE: 'bad\0value' } })).toThrow('NUL'); + expect(fs.existsSync(path.join(reporter.root, 'commands'))).toBe(false); + }); + + test('a command that never spawned stops without a cleanup failure', async () => { + const running = runner.start({ + command: path.join(root, process.platform === 'win32' ? 'missing.exe' : 'missing'), + args: [], + cwd: root, + env: {}, + category: 'phase', + }); + await expect(running.completed).rejects.toMatchObject({ code: 'ENOENT' }); + await expect(running.stop({ failed: true })).resolves.toBeUndefined(); + const result = await reporter.finish(); + expect(result.failures.filter(({ kind }) => kind === 'cleanup')).toEqual([]); + }); + + test('output past the log limit is reported once and stops the command', async () => { + const limited = new NodeDesktopCommandRunner({ reporter, shutdownTimeoutMs: 2000, maxLogBytes: 4096 }); + await expect( + limited.run(node("const line = 'x'.repeat(99) + '\\n'; setInterval(() => process.stdout.write(line.repeat(50)), 5);")), + ).rejects.toThrow('size limit'); + await limited.close(); + const result = await reporter.finish(); + expect(result.failures.filter(({ error }) => error.message.includes('size limit'))).toHaveLength(1); + }); + + test('terminate() force-stops a command that ignores a graceful stop', async () => { + const marker = path.join(root, 'ready'); + const running = runner.start( + node( + "process.on('SIGTERM', () => {}); require('node:fs').writeFileSync(process.argv[1], 'ready'); setInterval(() => {}, 1000);", + marker, + ), + ); + await waitUntil(() => fs.existsSync(marker)); + runner.terminate(); + await deadline( + running.completed.catch(() => 1), + 5000, + 'Force-stopping the fixture', + ); + await running.stop({ failed: true }); + }); + + describe('on Windows', () => { + // Emulated: the POSIX fixture starts normally, then stops through the Windows path against a mocked taskkill. + let restorePlatform: (() => void) | undefined; + const taskkill = jest.mocked(spawnSync); + const taskkillCalls = () => taskkill.mock.calls.filter(([command]) => command === 'taskkill.exe'); + const emulateWindows = () => { + const descriptor = Object.getOwnPropertyDescriptor(process, 'platform')!; + Object.defineProperty(process, 'platform', { ...descriptor, value: 'win32' }); + restorePlatform = () => Object.defineProperty(process, 'platform', descriptor); + }; + const respond = (status: number, stderr = ''): SpawnSyncReturns => + ({ pid: 0, output: [], stdout: '', stderr, status, signal: null }) as SpawnSyncReturns; + const worker = (source: string, ...values: string[]) => runner.start({ ...node(source, ...values), category: 'worker' }); + const leader = async () => { + const marker = path.join(root, 'leader.pid'); + const running = worker( + "require('node:fs').writeFileSync(process.argv[1], String(process.pid)); setInterval(() => {}, 1000);", + marker, + ); + await waitUntil(() => fs.existsSync(marker) && fs.readFileSync(marker, 'utf8').length > 0); + return { running, pid: Number(fs.readFileSync(marker, 'utf8')) }; + }; + afterEach(() => { + restorePlatform?.(); + restorePlatform = undefined; + taskkill.mockReset().mockImplementation(jest.requireActual('node:child_process').spawnSync); + }); + + test('never addresses a process ID after the leader exit is recorded, even while a descendant holds its output open', async () => { + const marker = path.join(root, 'pids.json'); + const running = worker( + "const { spawn } = require('node:child_process');" + + "const grandchild = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'inherit' });" + + 'grandchild.unref();' + + "require('node:fs').writeFileSync(process.argv[1], JSON.stringify([process.pid, grandchild.pid]));", + marker, + ); + await waitUntil(() => fs.existsSync(marker) && fs.readFileSync(marker, 'utf8').length > 0); + const [leaderPid, grandchildPid] = JSON.parse(fs.readFileSync(marker, 'utf8')) as [number, number]; + try { + // Once the leader is reaped, its exit is recorded while the grandchild still holds the output pipes open. + await waitUntil(() => !alive(leaderPid)); + emulateWindows(); + runner.terminate(); + await running.stop({ failed: true }).catch(() => undefined); + expect(taskkillCalls()).toEqual([]); + } finally { + restorePlatform?.(); + if (alive(grandchildPid)) process.kill(grandchildPid, 'SIGKILL'); + } + }); + + test('resolves the leader before this loop can record its exit, and trusts that record over a failed taskkill', async () => { + const { running, pid } = await leader(); + emulateWindows(); + // The leader exits just before taskkill resolves it, so taskkill reports that it found nothing. + taskkill.mockImplementation(((command: string) => { + if (command !== 'taskkill.exe') throw new Error(`Unexpected synchronous spawn: ${command}`); + if (alive(pid)) process.kill(pid, 'SIGKILL'); + return respond(128, 'ERROR: The process was not found.'); + }) as unknown as typeof spawnSync); + runner.terminate(); + // Synchronous: the exit could not have been recorded, so the process ID was still bound to the owned leader. + expect(taskkillCalls()).toEqual([ + ['taskkill.exe', ['/PID', String(pid), '/T', '/F'], expect.objectContaining({ windowsHide: true })], + ]); + await running.stop({ failed: true }); + }); + + test('a failed taskkill for a leader that keeps running fails the stop with its diagnostic', async () => { + const { running, pid } = await leader(); + emulateWindows(); + taskkill.mockImplementation((() => respond(1, 'ERROR: Access is denied.')) as unknown as typeof spawnSync); + try { + await expect(running.stop()).rejects.toThrow(`Could not stop owned process tree ${pid}: ERROR: Access is denied.`); + } finally { + restorePlatform?.(); + process.kill(-pid, 'SIGKILL'); + } + }); + }); +}); + +function alive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ESRCH') return false; + throw error; + } +} + +async function waitUntil(check: () => boolean): Promise { + await deadline( + (async () => { + while (!check()) await new Promise((resolve) => setTimeout(resolve, 20)); + })(), + 10_000, + 'Waiting for the fixture', + ); +} diff --git a/packages/agentic/desktop-test/src/commands.ts b/packages/agentic/desktop-test/src/commands.ts index 6407a365a4..49510fea32 100644 --- a/packages/agentic/desktop-test/src/commands.ts +++ b/packages/agentic/desktop-test/src/commands.ts @@ -1,4 +1,4 @@ -import { spawn as spawnNative } from 'node:child_process'; +import { spawn as spawnNative, spawnSync } from 'node:child_process'; import spawn from 'cross-spawn'; import fs from 'node:fs'; import path from 'node:path'; @@ -28,9 +28,11 @@ export type RunningDesktopCommand = { waitForReadiness?(expected: DesktopServiceReady, timeoutMs?: number, signal?: AbortSignal): Promise; stop(options?: { failed?: boolean }): Promise; }; +/** Cancellation for one command: an abort stops its process tree and reports the command failed. */ +export type DesktopCommandRunOptions = { signal?: AbortSignal }; export interface DesktopCommandRunner { - run(command: PreparedDesktopCommand): Promise; - start(command: PreparedDesktopCommand): RunningDesktopCommand; + run(command: PreparedDesktopCommand, options?: DesktopCommandRunOptions): Promise; + start(command: PreparedDesktopCommand, options?: DesktopCommandRunOptions): RunningDesktopCommand; } export type NodeDesktopCommandRunnerOptions = { reporter?: DesktopReportWriter; @@ -50,7 +52,7 @@ const MIN_ENVIRONMENT_SECRET_LENGTH = 8; export class NodeDesktopCommandRunner implements DesktopCommandRunner { private reporterValue?: DesktopReportWriter; private ownedReporter?: DesktopRunReporter; - private readonly active = new Set(); + private readonly active = new Map void>(); private readonly options: NodeDesktopCommandRunnerOptions; constructor(options: NodeDesktopCommandRunnerOptions = {}) { this.options = options; @@ -60,15 +62,16 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { return this.reporterValue; } - async run(command: PreparedDesktopCommand): Promise { - const running = this.start({ ...command, category: command.category ?? 'phase' }); + async run(command: PreparedDesktopCommand, options: DesktopCommandRunOptions = {}): Promise { + const running = this.start({ ...command, category: command.category ?? 'phase' }, options); let failure: unknown; try { const code = await running.completed; + if (options.signal?.aborted) throw options.signal.reason; if (code !== 0) throw running.failure ?? new Error(`Command "${commandLabel(command)}" exited with code ${code}. Log: ${running.logPath}`); } catch (error) { - failure = error; + failure = options.signal?.aborted ? options.signal.reason : error; } try { await running.stop({ failed: failure !== undefined }); @@ -78,7 +81,10 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { if (failure !== undefined) throw failure; } - start(command: PreparedDesktopCommand): RunningDesktopCommand { + start(command: PreparedDesktopCommand, options: DesktopCommandRunOptions = {}): RunningDesktopCommand { + const { signal } = options; + signal?.throwIfAborted(); + validatePreparedCommand(command); const reporter = (this.reporterValue ??= this.ownedReporter = @@ -93,6 +99,7 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { if (/nonce|token|password|secret|authorization/i.test(key) && value && value.length >= MIN_ENVIRONMENT_SECRET_LENGTH) reporter.addSecret(value); } + const timeoutMs = this.options.shutdownTimeoutMs ?? 10_000; const id = `command-${randomUUID()}`; const label = commandLabel(command); const emit = reporter.source(id); @@ -106,31 +113,59 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { emit({ kind: 'scope-start', payload: { label, category: command.category ?? 'service' } }, id, command.context); emit({ kind: 'artifact', payload: { path: relativeLog, description: label } }, id, command.context); let failed: unknown; - const processControl = createDesktopProcessControl((error) => { - failed ??= error; - reporter.fail(error, 'infrastructure', id); - }); - reporter.addSecret(processControl.nonce); + // Only services publish readiness or cooperate in disposal; one-shot phases and workers need no control channel. + const processControl = + command.category === undefined || command.category === 'service' + ? createDesktopProcessControl((error) => { + failed ??= error; + reporter.fail(error, 'infrastructure', id); + }) + : undefined; + if (processControl) reporter.addSecret(processControl.nonce); const environment = { ...process.env, ...command.env, ...this.options.environment }; const spawnCommand = needsWindowsCommandShim(command.command) ? spawn : spawnNative; - const child = spawnCommand(command.command, [...command.args], { - cwd: command.cwd, - env: { - ...environment, - ...(command.category === undefined || command.category === 'service' ? processControl.environment(environment.NODE_OPTIONS) : {}), - [DESKTOP_REPORT_SCOPE]: id, - }, - detached: process.platform !== 'win32', - stdio: ['ignore', 'pipe', 'pipe'], - windowsHide: true, - }); + const child = (() => { + try { + return spawnCommand(command.command, [...command.args], { + cwd: command.cwd, + env: { + ...environment, + ...(processControl ? processControl.environment(environment.NODE_OPTIONS) : {}), + [DESKTOP_REPORT_SCOPE]: id, + }, + detached: process.platform !== 'win32', + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }); + } catch (error) { + // A synchronous spawn failure leaves no process to own: close the scope and the control channel here. + const summary = describeFailure(error); + emit( + { kind: 'scope-end', payload: { status: 'failed', logPath: relativeLog, ...(summary ? { summary } : {}) } }, + id, + command.context, + ); + emit({ kind: 'producer-finished', payload: { status: 'failed' } }, id, command.context); + void processControl?.close().catch(() => undefined); + throw error; + } + })(); let stopped = false; + let exited = false; let closed = false; + let drainExpired = false; + let captureFailed = false; let exitFailure: Error | undefined; let exitSummary: string | undefined; let stopPromise: Promise | undefined; + let treeStopPromise: Promise | undefined; let closeDeadline: ReturnType | undefined; let logBytes = fs.statSync(logPath).size; + // A POSIX group outlives its leader while descendants remain, and its ID cannot be reused while any member exists. + // A Windows process ID may be reused once the leader is reaped, so a tree is only addressed while it runs. + const treeRunning = () => child.pid !== undefined && (process.platform === 'win32' ? !exited : !closed || drainExpired); + const stopTree = () => + (treeStopPromise ??= treeRunning() ? stopProcessTree(child.pid!, settledCompletion, timeoutMs) : Promise.resolve()); const capture = (stream: 'stdout' | 'stderr') => { const decoder = new StringDecoder('utf8'); let buffered = ''; @@ -156,12 +191,17 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { buffered = ''; } }; + // The first capture failure is the diagnosis; later output is drained without recording so the child cannot block. const onFailure = (error: unknown) => { + if (captureFailed) return; + captureFailed = true; + buffered = ''; failed ??= error; reporter.fail(error, 'reporting', id); - child.kill(); + void stopTree().catch(() => undefined); }; child[stream].on('data', (chunk: Buffer) => { + if (captureFailed) return; try { consume(decoder.write(chunk), false); } catch (error) { @@ -169,6 +209,7 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { } }); child[stream].on('end', () => { + if (captureFailed) return; try { consume(decoder.end(), true); } catch (error) { @@ -184,12 +225,14 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { failed ??= error; }); child.once('exit', () => { - processControl.stop(); + exited = true; + processControl?.stop(); closeDeadline = setTimeout(() => { + drainExpired = true; failed ??= new Error(`Child exited but inherited output streams did not close within the drain deadline: ${label}`); child.stdout.destroy(); child.stderr.destroy(); - }, this.options.shutdownTimeoutMs ?? 10_000); + }, timeoutMs); }); child.once('close', (code, signal) => { clearTimeout(closeDeadline); @@ -202,6 +245,11 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { else resolve(code ?? (signal ? 1 : 0)); }); }); + // Waiting for the process does not consume its failure; owners observe that through completed. + const settledCompletion = completion.then( + (code) => code, + () => 1, + ); // Owners observe completed; this branch only makes unexpected failures visible immediately. void completion.then( (code) => { @@ -209,58 +257,67 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { }, (error: unknown) => reporter.fail(error, 'infrastructure', id), ); + const onAbort = () => { + void running.stop({ failed: true }).catch(() => undefined); + }; const running: RunningDesktopCommand = { completed: completion, logPath, - waitForReadiness: (expected, timeoutMs, signal) => - Promise.race([ - processControl.waitForReadiness(expected, timeoutMs, signal), - completion.then(() => { - throw failed ?? exitFailure ?? new Error('Owned command exited before service readiness.'); - }), - ]), + ...(processControl + ? { + waitForReadiness: (expected: DesktopServiceReady, readinessTimeoutMs?: number, readinessSignal?: AbortSignal) => + Promise.race([ + processControl.waitForReadiness(expected, readinessTimeoutMs, readinessSignal), + completion.then(() => { + throw failed ?? exitFailure ?? new Error('Owned command exited before service readiness.'); + }), + ]), + } + : {}), get failure() { return failed ?? exitFailure; }, - stop: (options = {}) => + stop: (stopOptions = {}) => (stopPromise ??= (async () => { stopped = true; + signal?.removeEventListener('abort', onAbort); let cleanupError: unknown; + const recordCleanup = (error: unknown) => { + cleanupError ??= error; + reporter.fail(error, 'cleanup', id); + }; const wasRunning = !closed; - const cooperative = processControl.stop(); + const cooperative = processControl?.stop() ?? 0; try { const encoded = this.options.environment?.[DESKTOP_REPORT_CONTEXT] ?? command.env[DESKTOP_REPORT_CONTEXT] ?? process.env[DESKTOP_REPORT_CONTEXT]; - const controlled = await requestReportStop(id, encoded, this.options.shutdownTimeoutMs ?? 10_000); - await processControl.settled(this.options.shutdownTimeoutMs ?? 10_000); + const controlled = await requestReportStop(id, encoded, timeoutMs); + await processControl?.settled(timeoutMs); if ((controlled || cooperative) && !closed) - await deadline(completion, this.options.shutdownTimeoutMs ?? 10_000, 'Gracefully stopping owned Node services'); + await deadline(settledCompletion, timeoutMs, 'Gracefully stopping owned Node services'); } catch (error) { - cleanupError = error; - reporter.fail(error, 'cleanup', id); + recordCleanup(error); } - if (wasRunning && child.pid) { - if (!closed) { - try { - await stopProcessTree(child.pid, completion, this.options.shutdownTimeoutMs ?? 10_000); - } catch (error) { - cleanupError ??= error; - reporter.fail(error, 'cleanup', id); - } - } + try { + await stopTree(); + } catch (error) { + recordCleanup(error); } let code = 1; try { - code = await deadline(completion, this.options.shutdownTimeoutMs ?? 10_000, 'Draining command output'); + code = await deadline(settledCompletion, timeoutMs, 'Draining command output'); } catch (error) { failed ??= error; cleanupError ??= error; } const status = - options.failed || failed !== undefined || cleanupError !== undefined || (!wasRunning && code !== 0) ? 'failed' : 'passed'; - const reason = status === 'failed' ? (describeFailure(failed ?? cleanupError) ?? exitSummary) : undefined; + stopOptions.failed || failed !== undefined || cleanupError !== undefined || (!wasRunning && code !== 0) ? 'failed' : 'passed'; + const reason = + status === 'failed' + ? (describeFailure(failed ?? cleanupError) ?? exitSummary ?? (wasRunning ? 'Stopped before it finished.' : undefined)) + : undefined; emit( { kind: 'scope-end', payload: { status, logPath: relativeLog, ...(reason ? { summary: reason } : {}) } }, id, @@ -268,16 +325,32 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { ); emit({ kind: 'producer-finished', payload: { status } }, id, command.context); this.active.delete(running); - await processControl.close(); + try { + await processControl?.close(); + } catch (error) { + recordCleanup(error); + } if (cleanupError !== undefined) throw cleanupError; })()), }; - this.active.add(running); + this.active.set(running, () => { + if (treeRunning()) forceStopProcessTree(child.pid!); + }); + signal?.addEventListener('abort', onAbort, { once: true }); return running; } + /** + * Force-kills every active command's process tree without waiting. Owners still observe completion and stop their + * commands; this only shortens a graceful shutdown that a repeated interrupt asked to abandon. + */ + terminate(): void { + for (const kill of this.active.values()) kill(); + } + async close(): Promise { - const results = await Promise.allSettled([...this.active].map((command) => command.stop())); + // A command still active here was abandoned by its owner, so its outcome is not a success. + const results = await Promise.allSettled([...this.active.keys()].map((command) => command.stop({ failed: true }))); await this.ownedReporter?.finish(); const failures = results.flatMap((result) => (result.status === 'rejected' ? [result.reason as unknown] : [])); if (failures.length) throw new AggregateError(failures, 'Owned commands did not stop cleanly.'); @@ -294,6 +367,14 @@ function commandLabel(command: PreparedDesktopCommand): string { return first !== undefined && subcommandPattern.test(first) ? `${name} ${first}` : name; } +function validatePreparedCommand(command: PreparedDesktopCommand): void { + const values = [command.command, command.cwd, ...command.args, ...Object.entries(command.env).flat()]; + if (values.some((value) => typeof value !== 'string' || value.includes('\0'))) + throw new TypeError( + `Command "${commandLabel(command)}" has a missing or NUL-containing command, argument, directory, or environment value.`, + ); +} + function describeFailure(error: unknown): string | undefined { if (error === undefined) return undefined; return (error instanceof Error ? error.message : String(error)).split('\n', 1)[0]; @@ -307,54 +388,91 @@ function needsWindowsCommandShim(command: string): boolean { ); } -async function stopProcessTree(pid: number, completed: Promise, timeoutMs: number): Promise { +async function stopProcessTree(pid: number, completed: Promise, timeoutMs: number): Promise { if (process.platform === 'win32') { - const killer = spawnNative('taskkill.exe', ['/PID', String(pid), '/T', '/F'], { stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true }); - let diagnostic = ''; - killer.stdout.on('data', (data: Buffer) => { - diagnostic += data.toString(); - }); - killer.stderr.on('data', (data: Buffer) => { - diagnostic += data.toString(); - }); - const code = await deadline( - new Promise((resolve, reject) => { - killer.once('error', reject); - killer.once('close', resolve); - }), - timeoutMs, - `Stopping owned process tree ${pid}`, - ); - if (code !== 0) { - try { - process.kill(pid, 0); - } catch (error) { - if (isMissingProcess(error)) return; - throw error; - } - throw new Error(`Could not stop owned process tree ${pid}: ${diagnostic}`); - } - } else { + const { status, diagnostic } = killWindowsProcessTree(pid, timeoutMs); + // A failed taskkill usually means the leader exited before taskkill resolved it. The recorded completion decides, + // never a probe of a process ID that may since have been reused. try { - process.kill(-pid, 'SIGTERM'); + await deadline(completed, timeoutMs, `Waiting for owned process tree ${pid}`); } catch (error) { - if (!isMissingProcess(error)) throw error; + if (status !== 0) throw new Error(`Could not stop owned process tree ${pid}: ${diagnostic}`, { cause: error }); + throw error; } + return; + } + // The group is gone only when its leader was reaped and no descendant remains in it. + const stoppedWithin = async (limitMs: number) => { + const until = Date.now() + limitMs; + for (;;) { + if (!processGroupExists(pid)) return true; + if (Date.now() >= until) return false; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + }; + if (!signalProcessGroup(pid, 'SIGTERM') || (await stoppedWithin(timeoutMs))) return; + if (!signalProcessGroup(pid, 'SIGKILL') || (await stoppedWithin(timeoutMs))) return; + throw new Error(`Owned process group ${pid} survived SIGKILL.`); +} + +function forceStopProcessTree(pid: number): void { + if (process.platform === 'win32') { try { - await deadline(completed, timeoutMs, `Stopping owned process group ${pid}`); - } catch (error) { - try { - process.kill(-pid, 'SIGKILL'); - } catch (killError) { - if (!isMissingProcess(killError)) throw new AggregateError([error, killError], 'Owned process group did not stop.'); - } + killWindowsProcessTree(pid, 10_000); + } catch { + // The owner's graceful stop reports a tree it cannot stop. } + return; + } + try { + signalProcessGroup(pid, 'SIGKILL'); + } catch { + // The owner's graceful stop reports a group it cannot signal. + } +} + +/** + * Callers address a Windows tree only before its leader's exit is recorded. Windows reuses a process ID only after every + * handle to the process closes, and Node closes its child handle while processing the exit on this event loop, so a + * synchronous taskkill keeps the ID bound to the owned leader, even one that has just exited, until taskkill resolves it. + * This is not containment: taskkill /T finds descendants only through parent process IDs, so a descendant whose parent + * already exited escapes. Only a Job Object would contain it. + */ +function killWindowsProcessTree(pid: number, timeoutMs: number): { status: number | null; diagnostic: string } { + const result = spawnSync('taskkill.exe', ['/PID', String(pid), '/T', '/F'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: timeoutMs, + windowsHide: true, + }); + if (result.error) throw result.error; + return { status: result.status, diagnostic: `${result.stdout}${result.stderr}`.trim() }; +} + +function signalProcessGroup(pid: number, signal: NodeJS.Signals): boolean { + try { + process.kill(-pid, signal); + return true; + } catch (error) { + if (isMissingProcess(error)) return false; + throw error; + } +} + +function processGroupExists(pid: number): boolean { + try { + process.kill(-pid, 0); + return true; + } catch (error) { + if (isMissingProcess(error)) return false; + // A group this user cannot signal is not ours to declare stopped. + return true; } - await deadline(completed, timeoutMs, `Waiting for owned process tree ${pid}`); } function isMissingProcess(error: unknown): boolean { - return error instanceof Error && 'code' in error && error.code === 'ESRCH'; + // Not instanceof: a system error may come from another realm, such as a test sandbox. + return typeof error === 'object' && error !== null && 'code' in error && error.code === 'ESRCH'; } export async function deadline(promise: Promise, timeoutMs: number, label: string): Promise { diff --git a/packages/agentic/desktop-test/src/config.ts b/packages/agentic/desktop-test/src/config.ts index 6665817721..f04c8a0a93 100644 --- a/packages/agentic/desktop-test/src/config.ts +++ b/packages/agentic/desktop-test/src/config.ts @@ -59,6 +59,8 @@ export type DesktopWdioSuite = { specFileRetries?: number; bail?: number; timeoutMs?: number; + /** Bounds the whole WDIO launcher run; by default it scales with timeoutMs, spec groups and retries. */ + suiteTimeoutMs?: number; optional?: boolean; }; @@ -232,7 +234,19 @@ export function validateDesktopConfig(value: unknown): asserts value is DesktopC } keys( suite, - ['kind', 'framework', 'specs', 'hooks', 'readiness', 'maxInstances', 'specFileRetries', 'bail', 'timeoutMs', 'optional'], + [ + 'kind', + 'framework', + 'specs', + 'hooks', + 'readiness', + 'maxInstances', + 'specFileRetries', + 'bail', + 'timeoutMs', + 'suiteTimeoutMs', + 'optional', + ], `suite ${id}`, ); oneOf(suite.framework, ['jasmine', 'mocha', 'cucumber'], `${id}.framework`); @@ -245,6 +259,7 @@ export function validateDesktopConfig(value: unknown): asserts value is DesktopC if (suite.maxInstances !== undefined && suite.maxInstances !== 1) throw new TypeError('Desktop suites require maxInstances: 1.'); for (const field of ['specFileRetries', 'bail']) positive(suite[field], `${id}.${field}`, true); positive(suite.timeoutMs, `${id}.timeoutMs`); + positive(suite.suiteTimeoutMs, `${id}.suiteTimeoutMs`); if (suite.hooks !== undefined) text(suite.hooks, `${id}.hooks`); if (suite.readiness !== undefined) validateReadiness(suite.readiness); if (suite.optional !== undefined && typeof suite.optional !== 'boolean') throw new TypeError(`${id}.optional must be boolean.`); @@ -462,12 +477,17 @@ export function validateCommand(value: unknown): asserts value is DesktopCommand keys(command, ['command', 'args', 'cwd', 'env'], 'command'); text(command.command, 'command.command'); if (command.cwd !== undefined) text(command.cwd, 'command.cwd'); - if (command.args !== undefined && (!Array.isArray(command.args) || !command.args.every((arg) => typeof arg === 'string'))) { - throw new TypeError('command.args must be strings.'); + // A NUL cannot reach a process: spawn rejects it, so it is a configuration error, not a launch failure. + if ( + command.args !== undefined && + (!Array.isArray(command.args) || !command.args.every((arg) => typeof arg === 'string' && !arg.includes('\0'))) + ) { + throw new TypeError('command.args must be strings without NUL.'); } if (command.env !== undefined) for (const [key, value] of Object.entries(object(command.env, 'command.env'))) { - if (typeof value !== 'string') throw new TypeError(`command.env.${key} must be a string.`); + if (key.includes('\0') || typeof value !== 'string' || value.includes('\0')) + throw new TypeError(`command.env.${key.replaceAll('\0', '\\0')} must be a string without NUL.`); } } diff --git a/packages/agentic/desktop-test/src/contracts.contract.cjs b/packages/agentic/desktop-test/src/contracts.contract.cjs index 8296d38e4b..ba3781270a 100644 --- a/packages/agentic/desktop-test/src/contracts.contract.cjs +++ b/packages/agentic/desktop-test/src/contracts.contract.cjs @@ -5,6 +5,7 @@ const os = require('node:os'); const { pathToFileURL } = require('node:url'); const { Writable } = require('node:stream'); const { spawnSync } = require('node:child_process'); +const { inspect } = require('node:util'); const mode = process.argv[2]; const packageRoot = path.resolve(__dirname, '..'); @@ -465,7 +466,8 @@ async function main() { }); const slow = reporter({ kind: 'lifecycle', output: stuck, flushTimeoutMs: 20 }); slow.emit({ kind: 'diagnostic', payload: { level: 'info', message: 'slow sink' } }); - assert.equal((await slow.finish()).status, 'failed'); + // A stalled console is a presentation warning; it never changes the verdict. + assert.equal((await slow.finish()).status, 'passed'); stuck.destroy(); const failedWrite = reporter({ kind: 'lifecycle' }); const rename = fs.renameSync; @@ -497,16 +499,19 @@ async function main() { }); try { const final = await late.finish(); - assert.equal(final.status, 'failed'); + assert.equal(final.status, 'passed'); assert.equal(JSON.parse(fs.readFileSync(path.join(late.root, 'run.json'), 'utf8')).status, final.status); const events = fs .readFileSync(path.join(late.root, 'events.jsonl'), 'utf8') .trim() .split('\n') .map((line) => JSON.parse(line)); - assert.equal(events.at(-1).payload.status, final.status); - assert(fs.readFileSync(summaryPath, 'utf8').includes('failed')); - assert(fs.readFileSync(path.join(late.root, 'junit.xml'), 'utf8').includes('late terminal fixture')); + // The final console line is written after the verdict, so its failure is journaled after run-finished. + assert.equal(events.filter(({ kind }) => kind === 'run-finished').at(-1).payload.status, final.status); + assert.deepEqual(events.at(-1).payload, { level: 'warning', message: 'Console output stopped: late terminal fixture' }); + assert(fs.readFileSync(summaryPath, 'utf8').includes('passed')); + assert(!fs.readFileSync(path.join(late.root, 'junit.xml'), 'utf8').includes('late terminal fixture')); + assert(err.text().includes('[desktop] WARNING: Console output stopped')); } finally { if (priorSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY; else process.env.GITHUB_STEP_SUMMARY = priorSummary; @@ -885,6 +890,145 @@ async function main() { const result = await run.finish(); assert.equal(result.status, 'failed'); assert(err.text().includes('original metro failure')); + } else if (mode === 'lifecycle') { + const { createDesktopWorkloadClaim } = await import(lib('ownership.js')); + const alive = (pid) => { + try { + process.kill(pid, 0); + return true; + } catch (error) { + if (error && typeof error === 'object' && 'code' in error && error.code === 'ESRCH') return false; + throw error; + } + }; + const sleeper = (marker, label) => + script("require('node:fs').writeFileSync(process.argv[1], String(process.pid)); setInterval(()=>{},1000);", [marker], label); + + // In-flight cancellation: a stop request ends a running stage promptly and the stage is not reported as passed. + const cancelRun = reporter({ kind: 'lifecycle' }); + const cancelRunner = new NodeDesktopCommandRunner({ reporter: cancelRun, shutdownTimeoutMs: 2000 }); + const cancelScope = new DesktopServiceScope(cancelRunner, cancelRun); + const stageMarker = path.join(root, 'stage.pid'); + const stage = cancelScope.guard( + cancelRunner.run({ ...sleeper(stageMarker, 'stage'), category: 'phase' }, { signal: cancelScope.controller.signal }), + ); + await waitFor(() => fs.existsSync(stageMarker) && fs.readFileSync(stageMarker, 'utf8').length > 0); + const started = Date.now(); + cancelScope.requestStop(); + await assert.rejects(stage, DesktopStopRequested); + await cancelScope.close(); + assert(Date.now() - started < 5000, 'cancellation must not wait for the stage to finish'); + await waitFor(() => !alive(Number(fs.readFileSync(stageMarker, 'utf8')))); + await cancelRunner.close(); + await cancelRun.finish(); + const stageEnd = fs + .readFileSync(path.join(cancelRun.root, 'events.jsonl'), 'utf8') + .trim() + .split('\n') + .map((line) => JSON.parse(line)) + .find((event) => event.kind === 'scope-end'); + assert.equal(stageEnd.payload.status, 'failed', JSON.stringify(stageEnd)); + // A spawn cannot follow the abort. + assert.throws(() => cancelRunner.start(command('', 'late'), { signal: cancelScope.controller.signal }), DesktopStopRequested); + + // Spawn failure: a service that never started settles its resource, so the workload claim releases. + const spawnRunner = new NodeDesktopCommandRunner({ reporter: reporter({ kind: 'lifecycle' }), shutdownTimeoutMs: 2000 }); + const spawnScope = new DesktopServiceScope(spawnRunner); + const spawnLocks = path.join(root, 'spawn-locks'); + const spawnClaim = createDesktopWorkloadClaim({ + runId: 'spawn', + targetId: 'target', + targetInstanceId: 'instance', + directory: spawnLocks, + }); + spawnScope.setResourceObserver((resource) => spawnClaim.updateResource(resource)); + const missing = spawnScope.start('missing', { + command: path.join(root, process.platform === 'win32' ? 'missing.exe' : 'missing'), + args: [], + cwd: root, + env: {}, + }); + await assert.rejects(missing.completed, { code: 'ENOENT' }); + await assert.rejects(spawnScope.waitForStop(), { code: 'ENOENT' }); + await spawnScope.close(); + await spawnRunner.close(); + assert.deepEqual( + spawnClaim.record.resources.map(({ disposal }) => disposal), + ['settled'], + ); + spawnClaim.release(); + assert(!fs.existsSync(path.join(spawnLocks, 'interactive.lock'))); + + // Observer failure during close: every service is still stopped and the close reports the failure. + const observerRunner = new NodeDesktopCommandRunner({ reporter: reporter({ kind: 'lifecycle' }), shutdownTimeoutMs: 2000 }); + const observerScope = new DesktopServiceScope(observerRunner); + const first = path.join(root, 'first.pid'); + const second = path.join(root, 'second.pid'); + observerScope.start('first', sleeper(first, 'first')); + observerScope.start('second', sleeper(second, 'second')); + await waitFor(() => [first, second].every((file) => fs.existsSync(file) && fs.readFileSync(file, 'utf8').length > 0)); + observerScope.setResourceObserver((resource) => { + if (resource.disposal === 'pending' && observerScope.controller.signal.aborted) throw new Error('claim volume unavailable'); + }); + await assert.rejects( + observerScope.close(), + (error) => error instanceof AggregateError && error.errors.some(({ message }) => message === 'claim volume unavailable'), + ); + for (const file of [first, second]) assert(!alive(Number(fs.readFileSync(file, 'utf8'))), `${file} must be stopped`); + await observerRunner.close(); + } else if (mode === 'interrupt-cli') { + // Windows has no deliverable SIGINT for a child process; the in-process contract covers cancellation there. + if (process.platform !== 'win32') { + const { spawn } = require('node:child_process'); + const marker = path.join(root, 'stage.pid'); + const reports = path.join(root, 'reports'); + const configPath = path.join(root, 'desktop.config.mjs'); + fs.writeFileSync( + configPath, + `export default ${JSON.stringify({ + schemaVersion: 1, + targets: { + win32: { + host: { kind: 'installed', application: { executablePath: 'not-launched.exe' } }, + commands: { + prep: { + command: process.execPath, + args: [ + '-e', + "require('node:fs').writeFileSync(process.argv[1], String(process.pid)); setInterval(()=>{},1000);", + marker, + ], + }, + }, + }, + }, + reporting: { artifactsRoot: reports }, + })}`, + ); + const cli = spawn( + process.execPath, + [path.join(packageRoot, 'config', 'cli.cjs'), '--config', configPath, '--target', 'win32', 'prep'], + { + cwd: root, + env: { ...process.env, DESKTOP_REPORTER: 'plain' }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + let output = ''; + cli.stdout.on('data', (chunk) => (output += chunk)); + cli.stderr.on('data', (chunk) => (output += chunk)); + const exited = new Promise((resolve) => cli.once('exit', (code, signal) => resolve({ code, signal }))); + await waitFor(() => fs.existsSync(marker) && fs.readFileSync(marker, 'utf8').length > 0); + const started = Date.now(); + cli.kill('SIGINT'); + const exit = await exited; + assert(Date.now() - started < 10_000, 'an interrupted stage must not run to completion'); + assert.deepEqual(exit, { code: 1, signal: null }, output); + assert.throws(() => process.kill(Number(fs.readFileSync(marker, 'utf8')), 0), { code: 'ESRCH' }); + const [runDirectory] = fs.readdirSync(reports); + const result = JSON.parse(fs.readFileSync(path.join(reports, runDirectory, 'run.json'), 'utf8')); + assert.equal(result.status, 'incomplete', JSON.stringify(result)); + } } else if (mode === 'resources') { const { DesktopServiceScope } = await import(lib('services.js')); const { createDesktopWorkloadClaim, inspectDesktopWorkloadClaim } = await import(lib('ownership.js')); @@ -932,6 +1076,29 @@ async function main() { ]), /dependency cycle/, ); + // Without a directory, the claim lives in the per-user cache rather than the shared temporary directory. + const home = path.join(root, 'home'); + fs.mkdirSync(home); + const savedEnvironment = { HOME: process.env.HOME, LOCALAPPDATA: process.env.LOCALAPPDATA }; + process.env.HOME = home; + process.env.LOCALAPPDATA = home; + try { + const defaultClaim = createDesktopWorkloadClaim({ runId: 'default-run', targetId: 'target', targetInstanceId: 'instance' }); + const defaultDirectory = + process.platform === 'win32' + ? path.join(home, 'Microsoft', 'FluentUIReactNative', 'desktop-test', 'workload') + : process.platform === 'darwin' + ? path.join(home, 'Library', 'Caches', 'com.microsoft.fluentui-react-native.desktop-test', 'workload') + : path.join(home, '.cache', 'fluentui-react-native', 'desktop-test', 'workload'); + assert(fs.existsSync(path.join(defaultDirectory, 'interactive.lock'))); + defaultClaim.release(); + assert(!fs.existsSync(path.join(defaultDirectory, 'interactive.lock'))); + } finally { + for (const [key, value] of Object.entries(savedEnvironment)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } const ready = { tier: 'service-ready', service: 'fixture-service', @@ -1198,6 +1365,11 @@ async function main() { assert.equal(live.state, 'live'); assert.equal(unknown.state, 'unknown'); assert.equal(reused.state, 'pid-reused'); + assert.deepEqual( + fs.readdirSync(failedDirectory).filter((name) => name.endsWith('.tmp')), + [], + 'claim updates replace the record atomically', + ); assert.equal(fs.existsSync(ownerFile), true, 'inspection must never reclaim a claim'); assert.throws( () => @@ -1853,6 +2025,8 @@ async function main() { "describe('hook fixture',()=>{beforeAll(()=>{throw new Error('original setup failure');});it('does not execute',()=>{throw new Error('body must not run');});});"; } else if (mode === 'crash') { source = 'process.exit(17);'; + } else if (mode === 'spec-load-failure') { + source = "throw new Error('original spec load failure');"; } else if (mode === 'lane-physical') { source = [ "import fs from 'node:fs';", @@ -1994,13 +2168,29 @@ async function main() { await supervisor.close(); } const result = await run.finish(); - if (['hook-failure', 'config-hook-failure', 'config-load-failure', 'crash', 'bail', 'lane-blocked'].includes(mode)) { + if ( + ['hook-failure', 'config-hook-failure', 'config-load-failure', 'crash', 'spec-load-failure', 'bail', 'lane-blocked'].includes(mode) + ) { assert(failure, `hook failure must fail launcher\n${JSON.stringify(result)}\n${err.text()}\n${out.text()}`); assert.equal(result.status, 'failed'); assert(!out.text().includes('PASSED WDIO fixture'), out.text()); if (mode !== 'crash') assert(!result.failures.some(({ kind }) => kind === 'cleanup' || kind === 'reporting'), JSON.stringify(result)); if (mode === 'hook-failure') assert(JSON.stringify(result).includes('original setup failure'), JSON.stringify(result)); + if (mode === 'spec-load-failure') { + // A spec that throws while loading reports no tests, yet it failed rather than being merely incomplete. + assert.deepEqual( + result.work.map(({ status }) => status), + ['failed'], + JSON.stringify(result), + ); + assert(result.work[0].label.endsWith('spec.mjs'), JSON.stringify(result)); + const log = result.artifacts.find((artifact) => artifact.path.startsWith('commands/')); + assert( + log && fs.readFileSync(path.join(run.root, log.path), 'utf8').includes('original spec load failure'), + JSON.stringify(result), + ); + } if (mode === 'config-hook-failure') assert(JSON.stringify(result).includes('missing-readiness-marker'), JSON.stringify(result)); if (mode === 'config-load-failure') { assert(JSON.stringify(result).includes('original config import failure'), JSON.stringify(result)); @@ -2082,6 +2272,7 @@ function attemptOf(result, title) { } main().catch((error) => { - process.stderr.write(error.stack ?? String(error)); + // Includes AggregateError members and causes, such as the cleanup failure behind a disposal error. + process.stderr.write(inspect(error, { depth: 8 })); process.exitCode = 1; }); diff --git a/packages/agentic/desktop-test/src/contracts.test.ts b/packages/agentic/desktop-test/src/contracts.test.ts index 64e24bb49c..6d38d8bd14 100644 --- a/packages/agentic/desktop-test/src/contracts.test.ts +++ b/packages/agentic/desktop-test/src/contracts.test.ts @@ -13,6 +13,8 @@ describe('neutral desktop harness contracts', () => { 'transport', 'delegated-cli', 'services', + 'lifecycle', + 'interrupt-cli', 'resources', 'owned-service', 'agent', @@ -25,6 +27,7 @@ describe('neutral desktop harness contracts', () => { 'config-hook-failure', 'config-load-failure', 'crash', + 'spec-load-failure', 'bail', 'lane-physical', 'lane-accessible', diff --git a/packages/agentic/desktop-test/src/driverService.ts b/packages/agentic/desktop-test/src/driverService.ts index 6aadce96d5..34641f9cad 100644 --- a/packages/agentic/desktop-test/src/driverService.ts +++ b/packages/agentic/desktop-test/src/driverService.ts @@ -70,7 +70,14 @@ export async function startDesktopService( let publication: DesktopAgentPublication | undefined; try { const recovery = await resolveRecovery(options, artifact); - const target = { ...options.target, ...(await options.createTarget?.(desktopHost, recovery)), host: desktopHost }; + // Only the declared story context is taken: an integration cannot replace the target's identity or launch modes. + const added = await options.createTarget?.(desktopHost, recovery); + const target = { + ...options.target, + ...(added?.storyOrchestrator === undefined ? {} : { storyOrchestrator: added.storyOrchestrator }), + ...(added?.storyRootTestId === undefined ? {} : { storyRootTestId: added.storyRootTestId }), + host: desktopHost, + }; options.signal?.throwIfAborted(); server = await createDesktopDriverServer({ ...(authority ? { authority } : {}), diff --git a/packages/agentic/desktop-test/src/index.ts b/packages/agentic/desktop-test/src/index.ts index 44f242d5ce..7a4e5b82c6 100644 --- a/packages/agentic/desktop-test/src/index.ts +++ b/packages/agentic/desktop-test/src/index.ts @@ -30,6 +30,8 @@ export type { PublishDesktopAgentConnection, } from './agentConnection.js'; export { DesktopServiceScope, DesktopStopRequested } from './services.js'; +export { handleDesktopInterrupts } from './signals.js'; +export type { DesktopInterruptHandlers } from './signals.js'; export { defineDesktopConfig, loadDesktopConfig, resolveDesktopSpecs, resolveDesktopTestScope, validateDesktopConfig } from './config.js'; export type { DesktopConfig, DesktopTargetConfig, DesktopWdioSuite, ResolvedDesktopConfig, ResolvedDesktopTarget } from './config.js'; export type { DesktopIntegration } from './integration.js'; diff --git a/packages/agentic/desktop-test/src/localSocket.test.ts b/packages/agentic/desktop-test/src/localSocket.test.ts new file mode 100644 index 0000000000..f5a29a0ef2 --- /dev/null +++ b/packages/agentic/desktop-test/src/localSocket.test.ts @@ -0,0 +1,48 @@ +import fs from 'node:fs'; +import net from 'node:net'; +import os from 'node:os'; +import path from 'node:path'; +import { once } from 'node:events'; + +import { createLocalSocketAddress } from './localSocket.js'; + +const posix = process.platform === 'win32' ? test.skip : test; + +describe('local socket addresses', () => { + const originalTmpdir = process.env.TMPDIR; + let root: string; + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-local-socket-')); + }); + afterEach(() => { + if (originalTmpdir === undefined) delete process.env.TMPDIR; + else process.env.TMPDIR = originalTmpdir; + fs.rmSync(root, { force: true, recursive: true }); + }); + + test('rejects names that are not short identifiers', () => { + expect(() => createLocalSocketAddress('../escape', 'id')).toThrow('Invalid local socket name'); + expect(() => createLocalSocketAddress('a'.repeat(33), 'id')).toThrow('Invalid local socket name'); + }); + + posix('falls back to a short private directory when the temporary directory is too long to listen in', async () => { + const long = path.join(root, 'd'.repeat(120)); + fs.mkdirSync(long); + process.env.TMPDIR = long; + const endpoint = createLocalSocketAddress('process', 'unused'); + const directory = path.dirname(endpoint.address); + try { + expect(Buffer.byteLength(endpoint.address)).toBeLessThanOrEqual(103); + expect(endpoint.address.startsWith(long)).toBe(false); + expect(fs.statSync(directory).mode & 0o777).toBe(0o700); + const server = net.createServer(); + server.listen(endpoint.address); + await once(server, 'listening'); + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + } finally { + endpoint.dispose(); + endpoint.dispose(); + } + expect(fs.existsSync(directory)).toBe(false); + }); +}); diff --git a/packages/agentic/desktop-test/src/localSocket.ts b/packages/agentic/desktop-test/src/localSocket.ts new file mode 100644 index 0000000000..233305e36c --- /dev/null +++ b/packages/agentic/desktop-test/src/localSocket.ts @@ -0,0 +1,46 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +export type LocalSocketAddress = { + /** A Windows named pipe, or a Unix-domain socket path inside a private directory. */ + readonly address: string; + /** Removes the private socket directory. Call it after the server closes. */ + dispose(): void; +}; + +// sockaddr_un.sun_path holds 104 bytes on macOS and the BSDs and 108 on Linux, including the terminating NUL. +const unixSocketPathLimit = process.platform === 'linux' ? 107 : 103; +const namePattern = /^[a-z][a-z0-9-]{0,31}$/; +const directoryPrefix = 'furn-'; + +/** + * Reserves a short local IPC address. POSIX sockets go in a new owner-only directory under the temporary directory, or + * under `/tmp` when the temporary directory is too long for the platform's socket-path limit. + */ +export function createLocalSocketAddress(name: string, unique: string): LocalSocketAddress { + if (!namePattern.test(name)) throw new TypeError('Invalid local socket name.'); + if (process.platform === 'win32') { + if (!/^[\w-]{1,64}$/.test(unique)) throw new TypeError('Invalid local pipe identity.'); + return { address: `\\\\.\\pipe\\furn-${name}-${unique}`, dispose: () => undefined }; + } + const socketName = `${name}.sock`; + // mkdtemp appends six characters to the prefix. + const fits = (root: string) => Buffer.byteLength(path.join(root, `${directoryPrefix}XXXXXX`, socketName)) <= unixSocketPathLimit; + const root = [os.tmpdir(), '/tmp'].find(fits); + if (!root) + throw new Error( + `Local socket paths under ${os.tmpdir()} exceed the ${unixSocketPathLimit}-byte platform limit; set TMPDIR to a shorter directory.`, + ); + const directory = fs.mkdtempSync(path.join(root, directoryPrefix)); + fs.chmodSync(directory, 0o700); + let disposed = false; + return { + address: path.join(directory, socketName), + dispose() { + if (disposed) return; + disposed = true; + fs.rmSync(directory, { recursive: true, force: true }); + }, + }; +} diff --git a/packages/agentic/desktop-test/src/ownedService.test.ts b/packages/agentic/desktop-test/src/ownedService.test.ts new file mode 100644 index 0000000000..a752591852 --- /dev/null +++ b/packages/agentic/desktop-test/src/ownedService.test.ts @@ -0,0 +1,72 @@ +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +jest.setTimeout(30_000); + +const posixTest = process.platform === 'win32' ? test.skip : test; + +describe('owned desktop service interrupts', () => { + posixTest('a second interrupt during cleanup force-stops a service that ignores the graceful stop', async () => { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'furn-owned-service-'))); + const marker = path.join(root, 'service.pid'); + const descriptor = path.join(root, 'service.json'); + // Fixture code is constant; the marker path reaches it through process.argv (R-SCAN-002). + fs.writeFileSync( + descriptor, + JSON.stringify({ + command: { + command: process.execPath, + args: [ + '-e', + "process.on('SIGTERM', () => {}); require('node:fs').writeFileSync(process.argv[1], String(process.pid)); setInterval(() => {}, 1000);", + marker, + ], + cwd: root, + }, + stopPath: path.join(root, 'service.stop'), + label: 'fixture', + }), + ); + let servicePid: number | undefined; + const child = spawn(process.execPath, [path.resolve(__dirname, '../config/service.cjs'), descriptor], { cwd: root, stdio: 'ignore' }); + const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => + child.once('exit', (code, signal) => resolve({ code, signal })), + ); + try { + await waitUntil(() => fs.existsSync(marker) && fs.readFileSync(marker, 'utf8').length > 0); + servicePid = Number(fs.readFileSync(marker, 'utf8')); + child.kill('SIGINT'); + // The graceful stop is still waiting on the service when the second interrupt arrives. + await new Promise((resolve) => setTimeout(resolve, 500)); + expect(isAlive(servicePid)).toBe(true); + const escalated = Date.now(); + child.kill('SIGINT'); + expect((await exited).signal).toBeNull(); + expect(Date.now() - escalated).toBeLessThan(8_000); + await waitUntil(() => !isAlive(servicePid!)); + } finally { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + if (servicePid && isAlive(servicePid)) process.kill(servicePid, 'SIGKILL'); + fs.rmSync(root, { force: true, recursive: true, maxRetries: 20, retryDelay: 100 }); + } + }); +}); + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'EPERM'; + } +} + +async function waitUntil(condition: () => boolean, timeoutMs = 15_000): Promise { + const deadline = Date.now() + timeoutMs; + while (!condition()) { + if (Date.now() > deadline) throw new Error('Timed out waiting for the condition.'); + await new Promise((resolve) => setTimeout(resolve, 50)); + } +} diff --git a/packages/agentic/desktop-test/src/ownedService.ts b/packages/agentic/desktop-test/src/ownedService.ts index 5f0ce59a94..41c2080e53 100644 --- a/packages/agentic/desktop-test/src/ownedService.ts +++ b/packages/agentic/desktop-test/src/ownedService.ts @@ -2,8 +2,10 @@ import fs from 'node:fs'; import path from 'node:path'; import { validateCommand } from './config.js'; import { NodeDesktopCommandRunner } from './commands.js'; +import type { RunningDesktopCommand } from './commands.js'; import { createDesktopReportSession } from './reporting/session.js'; import { finishDesktopProcess, registerDesktopProcessDisposal } from './processControl.js'; +import { handleDesktopInterrupts } from './signals.js'; /** Bridges a generated PowerShell service descriptor into the parent's resource/reporting scope. */ export async function runOwnedDesktopService(descriptorPath: string): Promise { @@ -52,12 +54,12 @@ async function runService(descriptorPath: string, signal: AbortSignal): Promise< if (fs.existsSync(stopPath)) stop(); }, 100); report.onStop?.(stop); - process.once('SIGINT', stop); - process.once('SIGTERM', stop); + const releaseInterrupts = handleDesktopInterrupts({ stop, terminate: () => runner.terminate() }); let failure: unknown; + let running: RunningDesktopCommand | undefined; try { signal.throwIfAborted(); - const running = runner.start({ + running = runner.start({ ...command, cwd, args: command.args ?? [], @@ -66,10 +68,11 @@ async function runService(descriptorPath: string, signal: AbortSignal): Promise< label: value.label, context: { service: value.label }, }); + const service = running; await Promise.race([ stopped, - running.completed.then((code) => { - throw running.failure ?? new Error(`Required service "${value.label}" exited unexpectedly with code ${code}.`); + service.completed.then((code) => { + throw service.failure ?? new Error(`Required service "${value.label}" exited unexpectedly with code ${code}.`); }), ]); } catch (error) { @@ -77,16 +80,22 @@ async function runService(descriptorPath: string, signal: AbortSignal): Promise< report.reporter.emit({ kind: 'service-failed', payload: { failure: report.reporter.failure(error) } }, value.label); } finally { clearInterval(interval); - process.off('SIGINT', stop); - process.off('SIGTERM', stop); signal.removeEventListener('abort', stop); + // A requested stop is this service's normal end; runner.close() would report it as abandoned. + const failed = failure !== undefined; + for (const cleanup of [() => running?.stop({ failed }), () => runner.close()]) { + try { + await cleanup(); + } catch (error) { + report.reporter.fail(error, 'cleanup'); + failure = failure === undefined ? error : new AggregateError([failure, error], 'Service execution and cleanup failed.'); + } + } try { - await runner.close(); - } catch (error) { - report.reporter.fail(error, 'cleanup'); - failure = failure === undefined ? error : new AggregateError([failure, error], 'Service execution and cleanup failed.'); + process.exitCode = await report.close(failure); + } finally { + releaseInterrupts(); } - process.exitCode = await report.close(failure); } if (failure !== undefined) throw failure; } diff --git a/packages/agentic/desktop-test/src/ownership.ts b/packages/agentic/desktop-test/src/ownership.ts index 322eb5d65b..a22e45e7cf 100644 --- a/packages/agentic/desktop-test/src/ownership.ts +++ b/packages/agentic/desktop-test/src/ownership.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { createHash, randomUUID } from 'node:crypto'; +import { randomUUID } from 'node:crypto'; import type { DesktopResourceRecord } from './resources.js'; @@ -51,9 +51,23 @@ type DesktopWorkloadClaimOptions = { directory?: string; }; +/** + * The per-user claim directory. It is independent of TMPDIR, so workloads started with different temporary directories, + * and from different checkouts, contend for the same claim. + */ +function defaultDesktopWorkloadDirectory(platform: NodeJS.Platform = process.platform): string { + if (platform === 'win32') { + const localAppData = process.env.LOCALAPPDATA; + if (!localAppData) throw new Error('LOCALAPPDATA is required to resolve the per-user desktop workload claim directory.'); + return path.join(localAppData, 'Microsoft', 'FluentUIReactNative', 'desktop-test', 'workload'); + } + if (platform === 'darwin') + return path.join(os.homedir(), 'Library', 'Caches', 'com.microsoft.fluentui-react-native.desktop-test', 'workload'); + return path.join(os.homedir(), '.cache', 'fluentui-react-native', 'desktop-test', 'workload'); +} + export function createDesktopWorkloadClaim(options: DesktopWorkloadClaimOptions): DesktopWorkloadClaim { - const user = createHash('sha256').update(os.homedir()).digest('hex').slice(0, 16); - const root = options.directory ?? path.join(os.tmpdir(), `furn-desktop-workload-${user}`); + const root = options.directory ?? defaultDesktopWorkloadDirectory(); fs.mkdirSync(root, { recursive: true, mode: 0o700 }); const filename = path.join(root, 'interactive.lock'); const owner: DesktopWorkloadOwnerIdentity = Object.freeze({ @@ -85,7 +99,8 @@ export function createDesktopWorkloadClaim(options: DesktopWorkloadClaimOptions) if (fs.readFileSync(filename, 'utf8') !== contents) throw new Error('Desktop claim ownership changed; refusing to update another owner.'); const nextContents = JSON.stringify(next); - fs.writeFileSync(filename, nextContents, { mode: 0o600 }); + // A crash mid-write must leave the previous complete record, never a truncated one. + replaceFileAtomically(filename, nextContents); contents = nextContents; current = Object.freeze(next); }; @@ -159,6 +174,27 @@ export async function inspectDesktopWorkloadClaim( return { state: 'live', record }; } +function replaceFileAtomically(filename: string, contents: string): void { + const temporary = `${filename}.${randomUUID()}.tmp`; + fs.writeFileSync(temporary, contents, { flag: 'wx', mode: 0o600 }); + try { + for (let retries = 0; ; retries++) { + try { + fs.renameSync(temporary, filename); + return; + } catch (error) { + const code = typeof error === 'object' && error !== null && 'code' in error ? error.code : undefined; + if (process.platform !== 'win32' || retries === 10 || (code !== 'EPERM' && code !== 'EACCES' && code !== 'EBUSY')) throw error; + // Windows readers can briefly deny delete sharing; the previous record stays intact while the rename is retried. + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 25); + } + } + } catch (error) { + fs.rmSync(temporary, { force: true }); + throw error; + } +} + function parseOwnershipRecord(contents: string): DesktopWorkloadOwnershipRecord { const record = JSON.parse(contents) as DesktopWorkloadOwnershipRecord; if ( diff --git a/packages/agentic/desktop-test/src/processControl.ts b/packages/agentic/desktop-test/src/processControl.ts index fe8c19a4da..40340412a5 100644 --- a/packages/agentic/desktop-test/src/processControl.ts +++ b/packages/agentic/desktop-test/src/processControl.ts @@ -1,9 +1,8 @@ import net from 'node:net'; -import os from 'node:os'; -import path from 'node:path'; import { randomUUID, timingSafeEqual } from 'node:crypto'; import { once } from 'node:events'; import { processLifelineUrl } from '../config/process-path.cjs'; +import { createLocalSocketAddress } from './localSocket.js'; export type DesktopServiceReady = { tier: 'service-ready'; @@ -42,8 +41,8 @@ export function finishDesktopProcess(): void { export function createDesktopProcessControl(onFailure: (error: Error) => void) { const incarnation = randomUUID(); const nonce = randomUUID(); - const address = - process.platform === 'win32' ? `\\\\.\\pipe\\furn-process-${incarnation}` : path.join(os.tmpdir(), `furn-process-${incarnation}.sock`); + const endpoint = createLocalSocketAddress('process', incarnation); + const address = endpoint.address; const sockets = new Set(); const records = new Map(); const changed = new Set<() => void>(); @@ -200,9 +199,13 @@ export function createDesktopProcessControl(onFailure: (error: Error) => void) { return (closePromise ??= (async () => { stopping = true; for (const notify of changed) notify(); - await listening; - for (const socket of sockets) socket.destroy(); - await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + try { + await listening; + for (const socket of sockets) socket.destroy(); + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + } finally { + endpoint.dispose(); + } })()); }, }; diff --git a/packages/agentic/desktop-test/src/reporting/collector.test.ts b/packages/agentic/desktop-test/src/reporting/collector.test.ts index 95236ca1aa..b62ba27c76 100644 --- a/packages/agentic/desktop-test/src/reporting/collector.test.ts +++ b/packages/agentic/desktop-test/src/reporting/collector.test.ts @@ -4,8 +4,10 @@ import path from 'node:path'; import { randomUUID } from 'node:crypto'; import { spawn } from 'node:child_process'; import { once } from 'node:events'; +import { Writable } from 'node:stream'; import { DesktopRunReporter } from './collector.js'; +import { DesktopIncompleteError, DesktopReportedError } from './protocol.js'; import type { DesktopInputLaneReport } from './results.js'; describe('desktop run reporter input lanes', () => { @@ -539,3 +541,429 @@ describe('desktop run reporter operation output', () => { expect(out).toBe(`[desktop] PASSED: lifecycle (no tests); ${path.join(reporter.root, 'run.json')}\n`); }); }); + +describe('desktop run reporter verdicts', () => { + let artifactsRoot: string; + let out: string; + let err: string; + + beforeEach(() => { + artifactsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-verdicts-')); + out = ''; + err = ''; + }); + + afterEach(() => { + jest.restoreAllMocks(); + delete process.env.GITHUB_STEP_SUMMARY; + fs.rmSync(artifactsRoot, { force: true, recursive: true }); + }); + + type Options = Partial[0]>; + + function start(options: Options = {}): DesktopRunReporter { + return new DesktopRunReporter({ + artifactsRoot, + output: { write: (chunk: string) => ((out += chunk), true) }, + errorOutput: { write: (chunk: string) => ((err += chunk), true) }, + reporterMode: 'plain', + summary: false, + ...options, + }); + } + + function result( + reporter: DesktopRunReporter, + testId: string, + status: 'passed' | 'failed' | 'skipped' | 'unsupported' | 'incomplete', + attempt = 0, + parent?: string, + scope?: string, + ): void { + reporter.emit( + { + kind: 'test-result', + payload: { + testId, + title: testId, + attempt, + status, + durationMs: 1, + ...(status === 'failed' ? { failure: reporter.failure(new Error(`${testId} failed`), 'test') } : {}), + ...(status === 'passed' ? {} : { reason: `${testId} ${status}` }), + }, + }, + scope, + undefined, + parent, + ); + } + + /** A WDIO-style producer: selected work whose worker scope reports its tests (or none). */ + function worker(reporter: DesktopRunReporter, workId: string, testIds: string[] = []): void { + const source = `wdio-${randomUUID()}`; + const emit = reporter.source(source); + reporter.emit({ kind: 'work-selected', payload: { workId, label: workId } }); + emit({ kind: 'scope-start', payload: { label: `${workId} worker`, category: 'worker' } }, source, undefined, workId); + for (const testId of testIds) { + emit({ kind: 'test-result', payload: { testId, title: testId, attempt: 0, status: 'passed', durationMs: 1 } }, source); + } + emit({ kind: 'scope-end', payload: { status: 'passed' } }, source, undefined, workId); + emit({ kind: 'producer-finished', payload: { status: 'passed' } }, source); + } + + describe('retried tests', () => { + test.each(['skipped', 'unsupported', 'incomplete'] as const)( + 'a failed attempt followed only by a %s one stays failed', + async (status) => { + const reporter = start({ kind: 'tests' }); + result(reporter, 'one', 'passed'); + result(reporter, 'masked', 'failed', 0); + result(reporter, 'masked', status, 1); + const final = await reporter.finish(); + + expect(final.tests.find(({ testId }) => testId === 'masked')).toMatchObject({ status: 'failed', flaky: false }); + expect(final.status).toBe('failed'); + expect(final.counts).toMatchObject({ failed: 1, passed: 1, skipped: 0 }); + expect(fs.readFileSync(path.join(reporter.root, 'junit.xml'), 'utf8')).toContain(''); + }, + ); + + test('a retry that passes makes the test flaky and a skipped retry after a pass cannot undo it', async () => { + const reporter = start({ kind: 'tests' }); + result(reporter, 'flaky', 'failed', 0); + result(reporter, 'flaky', 'passed', 1); + const final = await reporter.finish(); + + expect(final.tests[0]).toMatchObject({ status: 'passed', flaky: true }); + expect(final.status).toBe('passed'); + }); + + test('attempts keep their arrival order whatever numbers the producer gives them', async () => { + const reporter = start({ kind: 'tests' }); + result(reporter, 'case', 'failed', 3_000_000); + result(reporter, 'case', 'failed', 1_000_000); + result(reporter, 'case', 'passed', 2_000_000); + const final = await reporter.finish(); + + expect(final.tests[0].attempts.map(({ attempt }) => attempt)).toEqual([3_000_000, 1_000_000, 2_000_000]); + expect(final.tests[0]).toMatchObject({ status: 'passed', flaky: true }); + }); + }); + + describe('selected work that produced no tests', () => { + test('an empty worker group is incomplete, with the reason, and cannot be asserted as covered', async () => { + const reporter = start({ kind: 'tests' }); + worker(reporter, 'covered', ['real-test']); + worker(reporter, 'empty'); + const final = await reporter.finish(); + + expect(final.status).toBe('incomplete'); + expect(final.work.find(({ workId }) => workId === 'empty')).toMatchObject({ + status: 'incomplete', + reason: 'No tests were reported for this work.', + }); + expect(final.work.find(({ workId }) => workId === 'covered')).toMatchObject({ status: 'passed' }); + expect(() => reporter.assertSuccessful(['empty'])).toThrow(DesktopIncompleteError); + expect(() => reporter.assertSuccessful(['covered'])).not.toThrow(); + expect(fs.readFileSync(path.join(reporter.root, 'junit.xml'), 'utf8')).toContain('No tests were reported for this work.'); + }); + + test('selected work that never started is a fault, not a coverage gap', async () => { + const reporter = start({ kind: 'tests' }); + reporter.emit({ kind: 'work-selected', payload: { workId: 'never', label: 'never ran' } }); + worker(reporter, 'covered', ['real-test']); + await reporter.finish(); + + expect(() => reporter.assertSuccessful(['never'])).toThrow('never ran: work is not-run'); + expect(() => reporter.assertSuccessful(['never'])).not.toThrow(DesktopIncompleteError); + }); + + test('a group that declared its exclusions ran what it was asked to, and phase work needs no tests', async () => { + const reporter = start({ kind: 'tests' }); + worker(reporter, 'covered', ['real-test']); + worker(reporter, 'filtered'); + reporter.emit({ kind: 'selection', payload: { excluded: { platform: 3, scope: 0, input: 0 }, key: 'filtered' } }); + reporter.emit({ kind: 'work-selected', payload: { workId: 'smoke', label: 'smoke' } }); + const phase = reporter.source('phase'); + phase({ kind: 'scope-start', payload: { label: 'smoke', category: 'phase' } }, 'phase', undefined, 'smoke'); + phase({ kind: 'scope-end', payload: { status: 'passed' } }, 'phase', undefined, 'smoke'); + phase({ kind: 'producer-finished', payload: { status: 'passed' } }, 'phase'); + const final = await reporter.finish(); + + expect(final.work.map(({ workId, status }) => [workId, status])).toEqual([ + ['covered', 'passed'], + ['filtered', 'passed'], + ['smoke', 'passed'], + ]); + expect(final.status).toBe('passed'); + }); + }); + + describe('typed incomplete errors', () => { + test('are the run verdict, not an infrastructure failure, when the run is already incomplete', async () => { + const reporter = start({ kind: 'tests' }); + worker(reporter, 'empty'); + const error = new DesktopIncompleteError('selected work produced no tests'); + reporter.fail(error); + reporter.fail(error); + const final = await reporter.finish(); + + expect(final.failures).toEqual([]); + expect(final.status).toBe('incomplete'); + expect(fs.readFileSync(path.join(reporter.root, 'events.jsonl'), 'utf8')).toContain('selected work produced no tests'); + }); + + test('become a failure when nothing else would stop the run from passing', async () => { + const reporter = start({ kind: 'lifecycle' }); + reporter.fail(new DesktopIncompleteError('a gap nothing else recorded')); + const final = await reporter.finish(); + + expect(final.status).toBe('failed'); + expect(final.failures).toHaveLength(1); + }); + + test('raised while other work is still running do not vanish when that work later passes', async () => { + const reporter = start({ kind: 'lifecycle' }); + const busy = reporter.source('busy'); + reporter.fail(new DesktopIncompleteError('a gap nothing else recorded')); + busy({ kind: 'producer-finished', payload: { status: 'passed' } }); + const final = await reporter.finish(); + + expect(final.status).toBe('failed'); + expect(final.failures).toHaveLength(1); + }); + + test('are relayed from a delegated reporter by name', async () => { + const reporter = start({ kind: 'tests' }); + worker(reporter, 'empty'); + const relayed = new DesktopReportedError({ + failureId: randomUUID(), + kind: 'infrastructure', + error: { name: 'DesktopIncompleteError', message: 'relayed incomplete' }, + }); + reporter.fail(relayed); + expect((await reporter.finish()).failures).toEqual([]); + }); + + test('reported by a producer as a failure are a warning only when the run already carries the verdict', async () => { + const incomplete = (reporter: DesktopRunReporter) => { + const emit = reporter.source('delegate'); + emit({ + kind: 'diagnostic', + payload: { + level: 'error', + message: 'no coverage', + failure: { failureId: randomUUID(), kind: 'infrastructure', error: { name: 'DesktopIncompleteError', message: 'no coverage' } }, + }, + }); + emit({ kind: 'producer-finished', payload: { status: 'failed' } }); + }; + const carried = start({ kind: 'tests' }); + worker(carried, 'empty'); + incomplete(carried); + expect(await carried.finish()).toMatchObject({ status: 'incomplete', failures: [] }); + + const alone = start({ kind: 'lifecycle' }); + incomplete(alone); + expect(await alone.finish()).toMatchObject({ status: 'failed', failures: [{ error: { name: 'DesktopIncompleteError' } }] }); + }); + }); + + describe('presentation errors', () => { + const closedPipe = () => + new Writable({ + write(_chunk, _encoding, callback) { + callback(Object.assign(new Error('write EPIPE'), { code: 'EPIPE' })); + }, + }); + const stalled = () => + new Writable({ + write(_chunk, _encoding, callback) { + setTimeout(callback, 300); + }, + }); + + test.each(['plain', 'github-actions'] as const)( + 'a %s consumer that closes the pipe cannot fail a passing run', + async (reporterMode) => { + const reporter = start({ kind: 'tests', reporterMode, output: closedPipe(), verbose: true }); + result(reporter, 'case', 'passed'); + const final = await reporter.finish(); + + expect(final.status).toBe('passed'); + expect(final.failures).toEqual([]); + expect(JSON.parse(fs.readFileSync(path.join(reporter.root, 'run.json'), 'utf8'))).toMatchObject({ + terminal: true, + status: 'passed', + }); + expect(err).toContain('[desktop] WARNING: Console output stopped: the consumer closed the stream (write EPIPE)'); + expect(err).toContain(`authoritative result: ${path.join(reporter.root, 'run.json')}`); + expect(err).not.toContain('FAILED'); + }, + ); + + test.each(['plain', 'github-actions'] as const)('a %s consumer that stalls cannot fail a passing run', async (reporterMode) => { + const reporter = start({ kind: 'tests', reporterMode, output: stalled(), flushTimeoutMs: 50, verbose: true }); + result(reporter, 'case', 'passed'); + const final = await reporter.finish(); + + expect(final.status).toBe('passed'); + expect(final.failures).toEqual([]); + expect(err).toContain('[desktop] WARNING: Console output stopped: Timed out flushing desktop report output.'); + }); + + test('a stall cannot hide a failure from the verdict or the retained report', async () => { + const reporter = start({ kind: 'tests', output: stalled(), flushTimeoutMs: 50, verbose: true }); + result(reporter, 'case', 'failed'); + const final = await reporter.finish(); + + expect(final.status).toBe('failed'); + expect(final.failures.map(({ kind }) => kind)).toEqual(['test']); + expect(JSON.parse(fs.readFileSync(path.join(reporter.root, 'run.json'), 'utf8'))).toMatchObject({ status: 'failed' }); + }); + + test('output past the queue bound is dropped with a warning and never changes the verdict', async () => { + const reporter = start({ kind: 'lifecycle', output: stalled(), flushTimeoutMs: 1_000, verbose: true }); + const source = reporter.source('noisy'); + const text = 'x'.repeat(256 * 1024) + '\n'; + for (let index = 0; index < 24; index++) source({ kind: 'output', payload: { stream: 'stdout', text } }, 'noisy'); + source({ kind: 'producer-finished', payload: { status: 'passed' } }, 'noisy'); + const final = await reporter.finish(); + + expect(final.status).toBe('passed'); + expect(final.failures).toEqual([]); + expect(err).toMatch(/\[desktop\] WARNING: \d+ console output items were dropped because the output queue exceeded 4 MiB/); + }); + + test('the final line comes from the terminal result, not from presentation state', async () => { + const reporter = start({ kind: 'tests' }); + result(reporter, 'case', 'passed'); + const final = await reporter.finish(); + + expect(final.status).toBe('passed'); + expect(out.split('\n').find((line) => line.startsWith('[desktop] PASSED:'))).toContain( + `${final.counts.selected} selected, ${final.counts.executed} executed, ${final.counts.passed} passed`, + ); + expect(out).not.toContain('FAILED'); + }); + }); + + describe('milestone events', () => { + function milestones(): { reporter: DesktopRunReporter; events: () => Record[] } { + const reporter = start({ kind: 'tests', reporterMode: 'ndjson', eventLevel: 'milestones' }); + return { + reporter, + events: () => + out + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as Record), + }; + } + const results = (events: Record[], status: string) => + events.filter(({ kind, payload }) => kind === 'test-result' && (payload as { status: string }).status === status); + + test('a flood of skips never suppresses a later failure, incomplete result, or cancellation', async () => { + const { reporter, events } = milestones(); + for (let index = 0; index < 40; index++) result(reporter, `skip-${index}`, 'skipped'); + for (let index = 0; index < 5; index++) result(reporter, `gap-${index}`, 'unsupported'); + result(reporter, 'broken', 'failed'); + result(reporter, 'unfinished', 'incomplete'); + await reporter.finish(); + + expect(results(events(), 'skipped')).toHaveLength(25); + expect(results(events(), 'failed')).toHaveLength(1); + expect(results(events(), 'incomplete')).toHaveLength(1); + expect(events().at(-1)).toMatchObject({ type: 'summary', status: 'failed' }); + }); + }); + + describe('retained reports', () => { + function runJsonWrites(): () => number { + const rename = fs.renameSync; + let writes = 0; + jest.spyOn(fs, 'renameSync').mockImplementation((source, destination) => { + if (String(destination).endsWith('run.json')) writes++; + return rename(source, destination); + }); + return () => writes; + } + + test('events that cannot change the result do not rewrite run.json', async () => { + const reporter = start({ kind: 'tests' }); + const writes = runJsonWrites(); + const source = reporter.source('noise'); + for (let index = 0; index < 30; index++) { + source({ kind: 'output', payload: { stream: 'stdout', text: `line ${index}\n` } }, 'noise'); + source({ kind: 'diagnostic', payload: { level: 'info', message: 'progress' } }, 'noise'); + await new Promise((resolve) => setImmediate(resolve)); + } + await reporter.flush(); + + expect(writes()).toBeLessThanOrEqual(1); + }); + + test('a burst of results is throttled, and flush and finish always write the current state', async () => { + const reporter = start({ kind: 'tests' }); + const writes = runJsonWrites(); + for (let index = 0; index < 40; index++) { + result(reporter, `case-${index}`, 'passed'); + await new Promise((resolve) => setImmediate(resolve)); + } + await reporter.flush(); + + expect(writes()).toBeLessThanOrEqual(3); + expect(JSON.parse(fs.readFileSync(path.join(reporter.root, 'run.json'), 'utf8'))).toMatchObject({ + terminal: false, + counts: { selected: 40 }, + }); + const final = await reporter.finish(); + expect(JSON.parse(fs.readFileSync(path.join(reporter.root, 'run.json'), 'utf8'))).toMatchObject({ + terminal: true, + counts: { selected: 40 }, + }); + expect(final.status).toBe('passed'); + }); + + test('a secret registered after its text was reported is redacted from every retained report', async () => { + const summary = path.join(artifactsRoot, 'summary.md'); + fs.writeFileSync(summary, ''); + process.env.GITHUB_STEP_SUMMARY = summary; + const reporter = start({ kind: 'tests', summary: 'auto' }); + reporter.emit({ + kind: 'test-result', + payload: { testId: 'case', title: 'logs in as late-secret-value', attempt: 0, status: 'passed', durationMs: 1 }, + }); + reporter.fail(new Error('token late-secret-value rejected'), 'test'); + reporter.addSecret('late-secret-value'); + await reporter.finish(); + + for (const file of ['run.json', 'junit.xml']) { + const text = fs.readFileSync(path.join(reporter.root, file), 'utf8'); + expect(text).not.toContain('late-secret-value'); + expect(text).toContain('[REDACTED]'); + } + expect(fs.readFileSync(summary, 'utf8')).not.toContain('late-secret-value'); + }); + }); + + describe('collector-only events', () => { + test('a producer cannot finish the run for the collector', () => { + const reporter = start({ kind: 'lifecycle' }); + expect(() => + reporter.accept({ + schemaVersion: 1, + runId: reporter.runId, + eventId: randomUUID(), + sourceId: 'rogue', + sourceSequence: 1, + scopeId: 'rogue', + timestamp: new Date().toISOString(), + kind: 'run-finished', + payload: { status: 'passed', resultPath: 'run.json' }, + }), + ).toThrow('A producer cannot report "run-finished".'); + }); + }); +}); diff --git a/packages/agentic/desktop-test/src/reporting/collector.ts b/packages/agentic/desktop-test/src/reporting/collector.ts index c6060d58a1..9a8701e44b 100644 --- a/packages/agentic/desktop-test/src/reporting/collector.ts +++ b/packages/agentic/desktop-test/src/reporting/collector.ts @@ -4,17 +4,31 @@ import { randomUUID } from 'node:crypto'; import { Writable } from 'node:stream'; import { formatOperationHeader, neutralizeWorkflowCommands, readLogExcerpt, singleLine, trimTrailingLineBreaks } from './presentation.js'; +import { DesktopFailureRegistry } from './failures.js'; import { createEventEmitter, + DesktopIncompleteError, DesktopReportedError, + isDesktopIncompleteFailure, redactDesktopSecrets, sanitizeDesktopReport, selectionExclusionReasons, - serializeReportError, validateDesktopEvent, } from './protocol.js'; -import type { CollectedDesktopEvent, EventContext, EventEmitter, FailureKind, ReportError, ReportFailure, TestStatus } from './protocol.js'; +import type { + CollectedDesktopEvent, + DesktopEvent, + EventContext, + EventEmitter, + FailureKind, + ReportError, + ReportFailure, + TestStatus, +} from './protocol.js'; import { + decideDesktopTestStatus, + decisiveDesktopAttempt, + describeDesktopIncompleteRun, describeDesktopSelection, describeDesktopUnsupported, projectDesktopRunSummary, @@ -26,10 +40,29 @@ import type { DesktopInputLaneReport, DesktopRunResult, DesktopTestResult, Deskt import { includeDesktopEvent, projectDesktopEvent } from './structured.js'; import type { DesktopEventLevel } from './structured.js'; import type { DesktopReportingConfig } from '../config.js'; -import { resolveDesktopReporterMode } from '@fluentui-react-native/desktop-driver/cli/reporter'; +import { isClosedDesktopStreamError, resolveDesktopReporterMode } from '@fluentui-react-native/desktop-driver/cli/reporter'; import type { ResolvedDesktopReporterMode } from '@fluentui-react-native/desktop-driver/cli/reporter'; const ROOT_SOURCE = 'supervisor'; +// The bound on console output waiting for a slow consumer. Past it, output is dropped with a warning; the run's result never changes. +const maximumPendingPresentationBytes = 4 * 1024 * 1024; +// A live run.json lags the run by at most this long; finalization and flush() always write the current state. +const snapshotIntervalMs = 250; +// Skips and declared gaps are progress detail in a milestones stream. A failed, incomplete, or cancelled result never is. +const maximumGapMilestones = 25; +const noTestsReason = 'No tests were reported for this work.'; +// Only events that can change the result need a new run.json; output and informational diagnostics cannot. +const snapshotKinds: ReadonlySet = new Set([ + 'work-selected', + 'scope-start', + 'scope-end', + 'test-start', + 'test-result', + 'selection', + 'artifact', + 'producer-finished', + 'service-failed', +]); // A failed operation shows the end of its log expanded. A passing one only collapses a longer excerpt into an Actions group. const expandedExcerpt = { lines: 200, bytes: 64 * 1024 }; const collapsedExcerpt = { lines: 1000, bytes: 256 * 1024 }; @@ -82,8 +115,15 @@ export class DesktopRunReporter { private readonly work = new Map(); private readonly tests = new Map(); private readonly testWork = new Map(); + // Work whose producer reports its tests through worker scopes, and the tests attributed to each. A worker group that + // ended without reporting any test (and without declaring exclusions) executed nothing, so it cannot count as covered. + private readonly workerWork = new Set(); + private readonly workTests = new Map>(); + private readonly excludedWork = new Set(); private readonly failures = new Map(); - private readonly failureIds = new WeakMap(); + /** Incomplete-run errors, which the run's status carries unless nothing else does. */ + private readonly verdicts = new Map(); + private readonly registry: DesktopFailureRegistry; private readonly annotations = new Set(); private readonly presentedFailures = new Set(); private readonly secrets = new Set(); @@ -104,12 +144,17 @@ export class DesktopRunReporter { private finishing?: Promise; private finished = false; private journalFailed = false; + // Presentation is best effort: a stalled, closed, or overloaded console is a warning and never changes the verdict. private presentationFailed = false; + private presenting = true; + private readonly presentationWarnings: string[] = []; + private droppedPresentation = 0; private suppressedEvents = 0; - private emittedProblemEvents = 0; - private structuredOutputError?: Error; + private emittedGapEvents = 0; private terminal?: DesktopRunResult; - private snapshotQueued = false; + private snapshotDirty = false; + private snapshotTimer?: NodeJS.Timeout; + private lastSnapshotAt = 0; constructor(options: DesktopRunReporterOptions) { this.options = options; @@ -130,7 +175,8 @@ export class DesktopRunReporter { this.output = options.output ?? process.stdout; this.errors = options.errorOutput ?? process.stderr; for (const secret of options.secrets ?? []) this.addSecret(secret); - this.emit = createEventEmitter(this.runId, ROOT_SOURCE, (event) => this.accept(event)); + this.registry = new DesktopFailureRegistry((value) => this.sanitize(value), this.failures); + this.emit = createEventEmitter(this.runId, ROOT_SOURCE, (event) => this.receive(event, true)); fs.writeFileSync(path.join(this.root, 'events.jsonl'), '', { flag: 'wx', mode: 0o600 }); this.artifacts.set('events.jsonl', 'Versioned event journal'); this.artifacts.set('run.json', 'Canonical run result'); @@ -146,14 +192,15 @@ export class DesktopRunReporter { ...(options.lane ? { lane: options.lane } : {}), artifactsRoot: clipStructuredText(this.root, 512), }; - this.enqueue(() => this.write(this.output, `${JSON.stringify(start)}\n`), 0); + const line = `${JSON.stringify(start)}\n`; + this.enqueue(() => this.write(this.output, line), Buffer.byteLength(line)); } } source(sourceId: string): EventEmitter { if (sourceId === ROOT_SOURCE || this.sources.has(sourceId)) throw new Error(`Report source "${sourceId}" is already registered.`); this.sources.set(sourceId, { sequence: 0, finished: false }); - return createEventEmitter(this.runId, sourceId, (event) => this.accept(event)); + return createEventEmitter(this.runId, sourceId, (event) => this.receive(event, false)); } addSecret(secret: string): void { @@ -165,42 +212,43 @@ export class DesktopRunReporter { } failure(error: unknown, kind: FailureKind = 'infrastructure'): ReportFailure { - if (error instanceof DesktopReportedError) return this.sanitize(error.failure); - let failureId = typeof error === 'object' && error !== null ? this.failureIds.get(error) : undefined; - if (!failureId && error instanceof Error && error.cause instanceof DesktopReportedError) return this.sanitize(error.cause.failure); - if (!failureId && error instanceof Error && error.cause && typeof error.cause === 'object') - failureId = this.failureIds.get(error.cause); - if (!failureId) { - failureId = randomUUID(); - if (typeof error === 'object' && error !== null) this.failureIds.set(error, failureId); - } - const recorded = this.failures.get(failureId); - if (recorded) return recorded; - return this.sanitize({ failureId, kind, error: serializeReportError(error) }); + return this.registry.failure(error, kind); } fail(error: unknown, kind: FailureKind = 'infrastructure', scopeId = 'run'): ReportFailure { - if (error instanceof AggregateError && error.errors.length) { - const children: unknown[] = error.errors; - return children.map((child) => this.fail(child, kind, scopeId))[0]; - } - if (error instanceof Error && error.cause instanceof AggregateError) return this.fail(error.cause, kind, scopeId); + return this.registry.fail( + error, + kind, + (failure) => { + try { + this.emit( + { kind: 'diagnostic', payload: { level: 'error', message: failure.error.message || failure.error.name, failure } }, + scopeId, + ); + } catch (reportingError) { + const reportingFailure = this.recordFailure(reportingError, 'reporting'); + this.print( + this.errors, + `Desktop reporting failed: ${reportingFailure.error.message}. Original failure: ${failure.error.message}. Reports: ${this.root}`, + ); + } + }, + (failure) => { + // The run's status carries an incomplete verdict, unless nothing else in the run does: then it is a failure. + this.verdicts.set(failure.failureId, failure); + try { + this.emit({ kind: 'diagnostic', payload: { level: 'warning', message: failure.error.message || failure.error.name } }, scopeId); + } catch (reportingError) { + this.recordFailure(reportingError, 'reporting'); + } + return true; + }, + ); + } + + private recordFailure(error: unknown, kind: FailureKind): ReportFailure { const failure = this.failure(error, kind); - if (this.failures.has(failure.failureId)) return failure; this.failures.set(failure.failureId, failure); - try { - this.emit( - { kind: 'diagnostic', payload: { level: 'error', message: failure.error.message || failure.error.name, failure } }, - scopeId, - ); - } catch (reportingError) { - const reportingFailure = this.failure(reportingError, 'reporting'); - this.failures.set(reportingFailure.failureId, reportingFailure); - this.print( - this.errors, - `Desktop reporting failed: ${reportingFailure.error.message}. Original failure: ${failure.error.message}. Reports: ${this.root}`, - ); - } return failure; } @@ -220,9 +268,14 @@ export class DesktopRunReporter { if (result.status === 'passed' || result.status === 'skipped') return; const failure = result.failures.find(({ kind }) => kind !== 'test') ?? - result.tests.find(({ status }) => status === 'failed')?.attempts.at(-1)?.failure; + decisiveDesktopAttempt(result.tests.find(({ status }) => status === 'failed')?.attempts ?? [])?.failure; if (failure) throw new DesktopReportedError(failure); - throw new Error(`Desktop run is ${result.status}; selected work did not produce complete executable results.`); + if (result.status !== 'incomplete') + throw new Error(`Desktop run is ${result.status}; selected work did not produce complete executable results.`); + const message = describeDesktopIncompleteRun(result); + // Selected work that never started means its producer did not deliver it, which is a fault rather than a coverage gap. + if (result.work.some(({ status }) => status === 'not-run')) throw new Error(message); + throw new DesktopIncompleteError(message); } artifactPath(relative: string): string { @@ -231,12 +284,19 @@ export class DesktopRunReporter { async flush(): Promise { await this.queue; + if (!this.finished) this.writePendingSnapshot(); } + /** Accepts an event from an external producer, which may never report a collector-only kind such as `run-finished`. */ accept(value: unknown): void { + this.receive(value, false); + } + + private receive(value: unknown, collector: boolean): void { if (this.finished) throw new Error('Cannot report after run finalization.'); - validateDesktopEvent(value); + validateDesktopEvent(value, collector ? 'collector' : 'producer'); if (value.runId !== this.runId) throw new Error('Report producer belongs to another run.'); + const knownSource = this.sources.has(value.sourceId); const source = this.sources.get(value.sourceId) ?? { sequence: 0, finished: false }; if (value.sourceSequence <= source.sequence) return; if (source.finished) throw new Error(`Producer ${value.sourceId} emitted after its terminal event.`); @@ -247,7 +307,7 @@ export class DesktopRunReporter { source.sequence = value.sourceSequence; source.finished = value.kind === 'producer-finished'; this.sources.set(value.sourceId, source); - const event: CollectedDesktopEvent = { ...this.sanitize(value), arrivalSequence: ++this.arrivalSequence }; + const event: CollectedDesktopEvent = { ...this.carryIncompleteVerdict(this.sanitize(value)), arrivalSequence: ++this.arrivalSequence }; if (event.kind === 'artifact') this.artifactPath(event.payload.path); if (event.kind === 'scope-end' && event.payload.logPath) this.artifactPath(event.payload.logPath); const line = JSON.stringify(event) + '\n'; @@ -259,26 +319,51 @@ export class DesktopRunReporter { fs.appendFileSync(path.join(this.root, 'events.jsonl'), line, { mode: 0o600 }); } catch (error) { this.journalFailed = true; - const failure = this.failure(error, 'reporting'); - this.failures.set(failure.failureId, failure); + this.recordFailure(error, 'reporting'); throw error; } } this.reduce(event); this.present(event); - if (!this.snapshotQueued) { - this.snapshotQueued = true; - queueMicrotask(() => { - this.snapshotQueued = false; - if (this.finished) return; - try { - this.persistSnapshot(this.result()); - } catch (error) { - const failure = this.failure(error, 'reporting'); - this.failures.set(failure.failureId, failure); - this.print(this.errors, `Cannot persist desktop run snapshot: ${failure.error.message}`); - } - }); + if (!knownSource || snapshotKinds.has(event.kind) || (event.kind === 'diagnostic' && event.payload.failure)) this.scheduleSnapshot(); + } + + /** + * A delegated process cannot see the run's status, so it reports an incomplete-run error as a failure. It is kept as a + * verdict instead: the final result counts it as a failure only if nothing else in the run carries the gap. + */ + private carryIncompleteVerdict(event: DesktopEvent): DesktopEvent { + if (event.kind !== 'diagnostic') return event; + const { failure, ...payload } = event.payload; + if (!failure || !isDesktopIncompleteFailure(failure)) return event; + this.verdicts.set(failure.failureId, failure); + return { ...event, payload: { ...payload, level: 'warning' } }; + } + + private scheduleSnapshot(): void { + this.snapshotDirty = true; + if (this.snapshotTimer) return; + this.snapshotTimer = setTimeout( + () => { + this.snapshotTimer = undefined; + if (!this.finished) this.writePendingSnapshot(); + }, + Math.max(0, this.lastSnapshotAt + snapshotIntervalMs - Date.now()), + ); + this.snapshotTimer.unref(); + } + + private writePendingSnapshot(): void { + if (this.snapshotTimer) clearTimeout(this.snapshotTimer); + this.snapshotTimer = undefined; + if (!this.snapshotDirty) return; + this.snapshotDirty = false; + this.lastSnapshotAt = Date.now(); + try { + this.persistSnapshot(this.result()); + } catch (error) { + const failure = this.recordFailure(error, 'reporting'); + this.print(this.errors, `Cannot persist desktop run snapshot: ${failure.error.message}`); } } @@ -290,7 +375,10 @@ export class DesktopRunReporter { } else if (event.kind === 'scope-start') { if (this.scopes.has(event.scopeId)) throw new Error(`Scope ${event.scopeId} was started twice.`); const work = event.parentScopeId ? this.work.get(event.parentScopeId) : undefined; - if (work) work.status = 'incomplete'; + if (work) { + work.status = 'incomplete'; + if (event.payload.category === 'worker') this.workerWork.add(work.workId); + } this.scopes.set(event.scopeId, { ...event.payload, finished: false, @@ -308,7 +396,12 @@ export class DesktopRunReporter { } else if (event.kind === 'test-start' || event.kind === 'test-result') { const p = event.payload; const workId = this.scopes.get(event.scopeId)?.workId ?? event.parentScopeId; - if (workId && this.work.has(workId)) this.testWork.set(p.testId, workId); + if (workId && this.work.has(workId)) { + this.testWork.set(p.testId, workId); + const attributed = this.workTests.get(workId) ?? new Set(); + attributed.add(p.testId); + this.workTests.set(workId, attributed); + } const test = this.tests.get(p.testId) ?? { testId: p.testId, title: p.title, status: 'incomplete', flaky: false, attempts: [] }; const index = test.attempts.findIndex(({ attempt }) => attempt === p.attempt); const result = @@ -327,14 +420,16 @@ export class DesktopRunReporter { throw new Error(`Duplicate terminal result for ${p.testId}, attempt ${p.attempt}.`); test.attempts[index] = result; } - test.attempts.sort((a, b) => a.attempt - b.attempt); - test.status = test.attempts[test.attempts.length - 1].status; + // Attempts stay in arrival order and the producer's attempt number is only a key, so a producer is free to number + // retries however it likes. The decisive attempt, not the last one, decides the test. + test.status = decideDesktopTestStatus(test.attempts); test.flaky = test.status === 'passed' && test.attempts.some(({ status }) => status === 'failed'); this.tests.set(p.testId, test); } else if (event.kind === 'selection') { this.excludedReported = true; - // A keyed event is idempotent: a retried worker announces the same exclusions again. + // A keyed event is idempotent: a retried worker announces the same exclusions again. The key names the work it covers. if (event.payload.key !== undefined) { + this.excludedWork.add(event.payload.key); if (this.selectionKeys.has(event.payload.key)) return; this.selectionKeys.add(event.payload.key); } @@ -343,16 +438,25 @@ export class DesktopRunReporter { } private present(event: CollectedDesktopEvent): void { + if (!this.presenting) return; if (this.structured) { if (this.reporterMode === 'ndjson') { const scopeCategory = this.scopes.get(event.scopeId)?.category; const eventLevel = this.options.eventLevel ?? 'milestones'; - const problemEvent = event.kind === 'test-result' && event.payload.status !== 'passed'; - if (eventLevel === 'milestones' && problemEvent && this.emittedProblemEvents >= 25) { + const gapEvent = event.kind === 'test-result' && (event.payload.status === 'skipped' || event.payload.status === 'unsupported'); + if (eventLevel === 'milestones' && gapEvent && this.emittedGapEvents >= maximumGapMilestones) { this.suppressedEvents++; } else if (includeDesktopEvent(event, eventLevel, scopeCategory)) { - if (problemEvent) this.emittedProblemEvents++; - this.enqueue(() => this.write(this.output, `${JSON.stringify(projectDesktopEvent(event))}\n`), 0); + if (gapEvent) this.emittedGapEvents++; + let line: string; + try { + line = `${JSON.stringify(projectDesktopEvent(event))}\n`; + } catch (error) { + this.suppressedEvents++; + this.warnPresentation(`An event could not be projected to NDJSON: ${describePresentationError(error)}`); + return; + } + if (!this.enqueue(() => this.write(this.output, line), Buffer.byteLength(line))) this.suppressedEvents++; } else { this.suppressedEvents++; } @@ -432,29 +536,28 @@ export class DesktopRunReporter { return sanitizeDesktopReport(value, (text) => this.redact(text)); } - private enqueue(action: () => Promise, bytes: number): void { - if (this.presentationFailed) return; - this.pendingBytes += bytes; - if (this.pendingBytes > 4 * 1024 * 1024) { - this.presentationFailed = true; - const failure = this.failure(new Error('Presentation queue exceeded 4 MiB; retained journal is authoritative.'), 'reporting'); - this.failures.set(failure.failureId, failure); - return; + /** Returns false when the output was dropped: the console is already failed, or the consumer is too far behind. */ + private enqueue(action: () => Promise, bytes: number): boolean { + if (this.presentationFailed) return false; + if (this.pendingBytes > 0 && this.pendingBytes + bytes > maximumPendingPresentationBytes) { + this.droppedPresentation++; + return false; } + this.pendingBytes += bytes; this.queue = this.queue .then(() => (this.presentationFailed ? undefined : action())) .catch((error: unknown) => { this.presentationFailed = true; - if (this.structured) { - this.structuredOutputError = error instanceof Error ? error : new Error(String(error)); - } else { - const failure = this.failure(error, 'reporting'); - this.failures.set(failure.failureId, failure); - } + this.warnPresentation(`${this.structured ? 'Structured' : 'Console'} output stopped: ${describePresentationError(error)}`); }) .finally(() => { this.pendingBytes -= bytes; }); + return true; + } + + private warnPresentation(message: string): void { + if (!this.presentationWarnings.includes(message)) this.presentationWarnings.push(message); } private async write(output: ReportOutput, text: string): Promise { @@ -556,6 +659,13 @@ export class DesktopRunReporter { } private result(): DesktopRunResult { + const result = this.build([...this.failures.values()]); + if (!this.verdicts.size || (result.status !== 'passed' && result.status !== 'skipped')) return result; + // Nothing else in the run carries the gap, so the error that reported it is the only record of it. + return this.build([...this.failures.values(), ...this.verdicts.values()]); + } + + private build(failures: ReportFailure[]): DesktopRunResult { return summarizeDesktopRun({ schemaVersion: 1, runId: this.runId, @@ -571,24 +681,46 @@ export class DesktopRunReporter { [...this.scopes.values()].some(({ finished }) => !finished) || [...this.sources.entries()].some(([id, source]) => id !== ROOT_SOURCE && !source.finished), tests: [...this.tests.values()], - work: [...this.work.values()], - failures: [...this.failures.values()], + work: [...this.work.values()].map( + (work): DesktopWorkResult => (this.reportedNoTests(work) ? { ...work, status: 'incomplete', reason: noTestsReason } : work), + ), + failures, artifacts: [...this.artifacts].map(([path, description]) => ({ path, description })), }); } - private persist(result: DesktopRunResult): void { - if (this.options.junit !== false) this.writeReport('junit.xml', renderJUnit(result)); - this.persistSnapshot(result); + // A finished worker group that attributed no test and declared no exclusions ran nothing: WDIO, for one, reports a spec + // that fails to load, or that defines no tests, as passed. + private reportedNoTests({ workId, status }: DesktopWorkResult): boolean { + return ( + this.workerWork.has(workId) && + (status === 'passed' || status === 'skipped' || status === 'unsupported') && + !this.workTests.get(workId)?.size && + !this.excludedWork.has(workId) + ); + } + + private persist(result: DesktopRunResult): DesktopRunResult { + // Secrets can be registered after an event is journaled, so the final reports redact against the complete set. + const safe = this.sanitize(result); + if (this.options.junit !== false) this.writeReport('junit.xml', renderJUnit(safe)); + this.writeSnapshot(safe); + return safe; } private persistSnapshot(result: DesktopRunResult): void { - const snapshot = this.sanitize({ - ...result, - terminal: result.terminal === true, - status: result.terminal === true || result.status === 'failed' ? result.status : 'incomplete', - }); - this.writeReport('run.json', JSON.stringify(snapshot) + '\n'); + this.writeSnapshot(this.sanitize(result)); + } + + private writeSnapshot(safe: DesktopRunResult): void { + this.writeReport( + 'run.json', + JSON.stringify({ + ...safe, + terminal: safe.terminal === true, + status: safe.terminal === true || safe.status === 'failed' ? safe.status : 'incomplete', + }) + '\n', + ); } private writeReport(name: string, data: string): void { @@ -624,17 +756,20 @@ export class DesktopRunReporter { } private async finalize(): Promise { + if (this.snapshotTimer) clearTimeout(this.snapshotTimer); + this.snapshotTimer = undefined; + this.snapshotDirty = false; await this.queue; - const persist = () => { - let result = this.result(); + // Writes the reports for the result `build` returns. A failed write is a reporting failure, which changes the result, so it is built again. + const persist = (build: () => DesktopRunResult): DesktopRunResult => { + let result = build(); try { - this.persist(result); + return this.persist(result); } catch (error) { - const failure = this.failure(error, 'reporting'); - this.failures.set(failure.failureId, failure); - result = this.result(); + this.recordFailure(error, 'reporting'); + result = build(); try { - this.persist(result); + return this.persist(result); } catch (retryError) { throw new AggregateError( [error, retryError, ...result.failures.map((failure) => new DesktopReportedError(failure))], @@ -642,9 +777,16 @@ export class DesktopRunReporter { ); } } - return result; }; - let result = persist(); + let result = persist(() => this.result()); + try { + this.emit({ kind: 'run-finished', payload: { status: result.status, resultPath: 'run.json' } }); + } catch (error) { + this.recordFailure(error, 'reporting'); + } + await this.queue; + // Nothing after this point can change the verdict: console output is best effort, so every line below describes this result. + result = persist(() => ({ ...this.result(), terminal: true })); const c = result.counts; const runResult = path.join(this.root, 'run.json'); this.print( @@ -655,17 +797,6 @@ export class DesktopRunReporter { ); const selection = describeDesktopSelection(result); if (selection) this.print(this.output, `[desktop] ${selection}`); - await this.queue; - result = persist(); - if (this.presentationFailed && !this.structured) { - try { - await this.writeLine(this.errors, `[desktop] FAILED: presentation could not be flushed; authoritative result: ${runResult}`); - } catch (error) { - const failure = this.failure(error, 'reporting'); - this.failures.set(failure.failureId, failure); - result = persist(); - } - } if (this.options.summary !== false && process.env.GITHUB_STEP_SUMMARY && !this.structured) { try { const summary = renderRunSummary(result).slice(0, 64 * 1024); @@ -673,26 +804,10 @@ export class DesktopRunReporter { if (size + Buffer.byteLength(summary) > 1024 * 1024) throw new Error('GitHub step summary would exceed 1 MiB.'); fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); } catch (error) { - this.emit({ - kind: 'diagnostic', - payload: { - level: 'warning', - message: `Optional Actions summary failed: ${String(error)}; result retained at ${path.join(this.root, 'run.json')}.`, - }, - }); + this.warnPresentation(`Optional Actions summary failed: ${String(error)}`); } } await this.queue; - result = persist(); - try { - this.emit({ kind: 'run-finished', payload: { status: result.status, resultPath: 'run.json' } }); - } catch (error) { - const failure = this.failure(error, 'reporting'); - this.failures.set(failure.failureId, failure); - result = persist(); - } - await this.queue; - result = persist(); const { suppressFinalOutput } = this.options; const suppressed = typeof suppressFinalOutput === 'function' ? suppressFinalOutput() : suppressFinalOutput === true; if (this.structured && !suppressed && !this.presentationFailed) { @@ -705,35 +820,43 @@ export class DesktopRunReporter { } await this.write(this.output, `${JSON.stringify(projectDesktopRunSummary(result, this.root))}\n`); } catch (error) { - this.structuredOutputError = error instanceof Error ? error : new Error(String(error)); + this.presentationFailed = true; + this.warnPresentation(`Structured output stopped: ${describePresentationError(error)}`); } } - if (this.structuredOutputError) { - const outputError = this.structuredOutputError; + await this.reportPresentationWarnings(runResult); + this.finished = true; + this.terminal = result; + return result; + } + + // Presentation problems reach the journal and stderr as warnings that point at the authoritative result. The journal + // write is best effort here: the terminal reports are already final, so a failure to record a warning cannot change them. + private async reportPresentationWarnings(runResult: string): Promise { + for (const message of this.presentationMessages()) { + this.presenting = false; try { - this.emit({ - kind: 'diagnostic', - payload: { level: 'warning', message: `Structured output stopped: ${outputError.message}` }, - }); - await this.queue; - result = this.result(); - persist(); - } catch (error) { - const failure = this.failure(error, 'reporting'); - this.failures.set(failure.failureId, failure); - result = persist(); + this.emit({ kind: 'diagnostic', payload: { level: 'warning', message } }); + } catch { + // The warning also goes to stderr below. + } finally { + this.presenting = true; } try { - await this.writeLine(this.errors, `[desktop] Structured output stopped: ${outputError.message}`); + await this.writeLine(this.errors, `[desktop] WARNING: ${message}; authoritative result: ${runResult}`); } catch { - // A closed stderr cannot carry the reporting diagnostic. + // A closed stderr cannot carry the warning. } } - result = { ...this.result(), terminal: true }; - this.persist(result); - this.finished = true; - this.terminal = result; - return result; + } + + private presentationMessages(): string[] { + return [ + ...this.presentationWarnings, + ...(this.droppedPresentation > 0 + ? [`${this.droppedPresentation} console output items were dropped because the output queue exceeded 4 MiB.`] + : []), + ]; } get resultSnapshot(): DesktopRunResult | undefined { @@ -741,6 +864,10 @@ export class DesktopRunReporter { } } +function describePresentationError(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + return isClosedDesktopStreamError(error) ? `the consumer closed the stream (${message})` : message; +} function errorDiagnostic(error: ReportError): string { const message = `${error.name}: ${error.message}`; return error.stack ? (error.stack.includes(error.message) ? error.stack : `${message}\n${error.stack}`) : message; diff --git a/packages/agentic/desktop-test/src/reporting/failures.test.ts b/packages/agentic/desktop-test/src/reporting/failures.test.ts new file mode 100644 index 0000000000..e0b0e47c3d --- /dev/null +++ b/packages/agentic/desktop-test/src/reporting/failures.test.ts @@ -0,0 +1,112 @@ +import { randomUUID } from 'node:crypto'; + +import { DesktopFailureRegistry } from './failures.js'; +import { DesktopIncompleteError, DesktopReportedError, redactDesktopSecrets, sanitizeDesktopReport } from './protocol.js'; +import type { ReportFailure } from './protocol.js'; + +describe('desktop failure registry', () => { + const secrets = new Set(); + const registry = () => new DesktopFailureRegistry((value) => sanitizeDesktopReport(value, (text) => redactDesktopSecrets(text, secrets))); + + afterEach(() => secrets.clear()); + + test('redacts a secret whose JSON encoding differs from its text', () => { + const secret = 'C:\\keys\\pa"ss'; + secrets.add(secret); + const failure = registry().failure(new Error(`cannot read ${secret}`), 'test'); + + expect(failure.error.message).toBe('cannot read [REDACTED]'); + expect(JSON.stringify(failure)).not.toContain('keys'); + expect(JSON.stringify(failure)).not.toContain('pa\\"ss'); + }); + + test('redacts a failure relayed from another process, and one registered after it was created', () => { + const relayed = new DesktopReportedError({ + failureId: randomUUID(), + kind: 'test', + error: { name: 'Error', message: 'late "quoted" secret', stack: 'Error: late "quoted" secret\n at here' }, + }); + secrets.add('late "quoted" secret'); + + expect(registry().failure(relayed)).toMatchObject({ error: { message: '[REDACTED]', stack: expect.not.stringContaining('quoted') } }); + }); + + test('records each error once, however many times or in what shape it is reported', () => { + const failures = registry(); + const recorded: ReportFailure[] = []; + const error = new Error('boom'); + const first = failures.fail( + error, + 'test', + (failure) => recorded.push(failure), + () => false, + ); + failures.fail( + error, + 'test', + (failure) => recorded.push(failure), + () => false, + ); + failures.fail( + new Error('wrapped', { cause: error }), + 'test', + (failure) => recorded.push(failure), + () => false, + ); + failures.fail( + new AggregateError([error], 'group'), + 'test', + (failure) => recorded.push(failure), + () => false, + ); + + expect(recorded).toEqual([first]); + expect([...failures.recorded.keys()]).toEqual([first.failureId]); + }); + + test('offers an incomplete verdict once and records it as a failure only when the run does not carry it', () => { + const failures = registry(); + const recorded: ReportFailure[] = []; + const offered: ReportFailure[] = []; + const carried = new DesktopIncompleteError('no coverage'); + failures.fail( + carried, + 'infrastructure', + (failure) => recorded.push(failure), + (failure) => (offered.push(failure), true), + ); + failures.fail( + carried, + 'infrastructure', + (failure) => recorded.push(failure), + (failure) => (offered.push(failure), true), + ); + + expect(offered).toHaveLength(1); + expect(recorded).toEqual([]); + expect(failures.recorded.size).toBe(0); + + const uncarried = new DesktopIncompleteError('nothing else says so'); + failures.fail( + uncarried, + 'infrastructure', + (failure) => recorded.push(failure), + () => false, + ); + expect(recorded).toHaveLength(1); + expect(recorded[0].error.name).toBe('DesktopIncompleteError'); + }); + + test('an error that only wraps an incomplete verdict is still a failure', () => { + const failures = registry(); + const recorded: ReportFailure[] = []; + failures.fail( + new Error('crash', { cause: new DesktopIncompleteError('inner') }), + 'infrastructure', + (failure) => recorded.push(failure), + () => true, + ); + + expect(recorded).toHaveLength(1); + }); +}); diff --git a/packages/agentic/desktop-test/src/reporting/failures.ts b/packages/agentic/desktop-test/src/reporting/failures.ts new file mode 100644 index 0000000000..9407c63360 --- /dev/null +++ b/packages/agentic/desktop-test/src/reporting/failures.ts @@ -0,0 +1,64 @@ +import { randomUUID } from 'node:crypto'; + +import { DesktopReportedError, isDesktopIncompleteError, serializeReportError } from './protocol.js'; +import type { FailureKind, ReportFailure } from './protocol.js'; + +/** + * The one place an error becomes a `ReportFailure`, shared by the collector and a delegated producer so both identify, + * deduplicate, and sanitize a failure the same way. Failures are sanitized as decoded values, never as encoded text. + */ +export class DesktopFailureRegistry { + private readonly ids = new WeakMap(); + private readonly acknowledged = new Set(); + private readonly sanitize: (value: T) => T; + /** Every failure recorded so far, by id. The collector passes the map its result is built from. */ + readonly recorded: Map; + + constructor(sanitize: (value: T) => T, recorded = new Map()) { + this.sanitize = sanitize; + this.recorded = recorded; + } + + failure(error: unknown, kind: FailureKind = 'infrastructure'): ReportFailure { + if (error instanceof DesktopReportedError) return this.sanitize(error.failure); + const object = typeof error === 'object' && error !== null ? error : undefined; + let failureId = object ? this.ids.get(object) : undefined; + if (!failureId && error instanceof Error && error.cause instanceof DesktopReportedError) return this.sanitize(error.cause.failure); + if (!failureId && error instanceof Error && error.cause && typeof error.cause === 'object') failureId = this.ids.get(error.cause); + if (!failureId) { + failureId = randomUUID(); + if (object) this.ids.set(object, failureId); + } + return this.recorded.get(failureId) ?? this.sanitize({ failureId, kind, error: serializeReportError(error) }); + } + + /** + * Records `error` once and returns its failure. `onFailure` receives each newly recorded failure. An incomplete-run + * verdict is offered once to `onIncomplete`, which returns true when the run's status already carries it; the error is + * then not a failure. If it returns false, the error is recorded as a failure like any other. + */ + fail( + error: unknown, + kind: FailureKind, + onFailure: (failure: ReportFailure) => void, + onIncomplete: (failure: ReportFailure) => boolean, + ): ReportFailure { + if (error instanceof AggregateError && error.errors.length) { + const children: unknown[] = error.errors; + return children.map((child) => this.fail(child, kind, onFailure, onIncomplete))[0]; + } + if (error instanceof Error && error.cause instanceof AggregateError) return this.fail(error.cause, kind, onFailure, onIncomplete); + const failure = this.failure(error, kind); + if (isDesktopIncompleteError(error)) { + if (this.acknowledged.has(failure.failureId)) return failure; + if (onIncomplete(failure)) { + this.acknowledged.add(failure.failureId); + return failure; + } + } + if (this.recorded.has(failure.failureId)) return failure; + this.recorded.set(failure.failureId, failure); + onFailure(failure); + return failure; + } +} diff --git a/packages/agentic/desktop-test/src/reporting/index.ts b/packages/agentic/desktop-test/src/reporting/index.ts index b5a82caa40..c87dd3f5f3 100644 --- a/packages/agentic/desktop-test/src/reporting/index.ts +++ b/packages/agentic/desktop-test/src/reporting/index.ts @@ -4,7 +4,14 @@ export { createDesktopReportSession } from './session.js'; export type { DesktopReportSession } from './session.js'; export { runDesktopOperation } from './operation.js'; export type { DesktopOperation, DesktopOperationOptions } from './operation.js'; -export { createEventEmitter, DesktopReportedError, serializeReportError, validateDesktopEvent } from './protocol.js'; +export { + createEventEmitter, + DesktopIncompleteError, + DesktopReportedError, + isDesktopIncompleteError, + serializeReportError, + validateDesktopEvent, +} from './protocol.js'; export type { DesktopEvent, EventContext, @@ -23,7 +30,10 @@ export type { DesktopEventLevel } from '@fluentui-react-native/desktop-driver/cl export { createDesktopInputLane, desktopInputEnvironment, parseDesktopInputLane, requestedDesktopInputLane } from './lane.js'; export type { DesktopInputLaneName, RequestedDesktopInputLane } from './lane.js'; export { + decideDesktopTestStatus, + decisiveDesktopAttempt, describeDesktopInputLane, + describeDesktopIncompleteRun, describeDesktopSelection, describeDesktopUnsupported, desktopInputLaneSources, diff --git a/packages/agentic/desktop-test/src/reporting/protocol.test.ts b/packages/agentic/desktop-test/src/reporting/protocol.test.ts index 786a29748c..10faaf39ff 100644 --- a/packages/agentic/desktop-test/src/reporting/protocol.test.ts +++ b/packages/agentic/desktop-test/src/reporting/protocol.test.ts @@ -1,4 +1,10 @@ -import { redactDesktopSecrets, validateDesktopEvent } from './protocol'; +import { + DesktopIncompleteError, + DesktopReportedError, + isDesktopIncompleteError, + redactDesktopSecrets, + validateDesktopEvent, +} from './protocol'; describe('declared gaps and selection events', () => { function event(kind: string, payload: Record): unknown { @@ -80,6 +86,32 @@ describe('declared gaps and selection events', () => { ])('rejects a selection with %s', (_name, excluded) => { expect(() => validateDesktopEvent(event('selection', { excluded }))).toThrow(); }); + + test('a producer cannot report the collector-only run-finished event', () => { + const finished = event('run-finished', { status: 'passed', resultPath: 'run.json' }); + expect(() => validateDesktopEvent(finished)).toThrow('A producer cannot report "run-finished".'); + expect(() => validateDesktopEvent(finished, 'collector')).not.toThrow(); + }); +}); + +describe('incomplete verdict errors', () => { + const relayed = (name: string) => + new DesktopReportedError({ failureId: 'failure', kind: 'infrastructure', error: { name, message: 'relayed' } }); + + test('recognizes the typed error, its relayed form, and an aggregate of only such errors', () => { + expect(isDesktopIncompleteError(new DesktopIncompleteError('nothing executed'))).toBe(true); + expect(isDesktopIncompleteError(relayed('DesktopIncompleteError'))).toBe(true); + expect(isDesktopIncompleteError(new AggregateError([new DesktopIncompleteError('a'), relayed('DesktopIncompleteError')]))).toBe(true); + }); + + test('never recognizes a fault, an aggregate that contains one, or an error that merely wraps the verdict', () => { + expect(isDesktopIncompleteError(new Error('boom'))).toBe(false); + expect(isDesktopIncompleteError(relayed('Error'))).toBe(false); + expect(isDesktopIncompleteError(new AggregateError([new DesktopIncompleteError('a'), new Error('b')]))).toBe(false); + expect(isDesktopIncompleteError(new Error('wrapped', { cause: new DesktopIncompleteError('a') }))).toBe(false); + expect(isDesktopIncompleteError(new AggregateError([]))).toBe(false); + expect(isDesktopIncompleteError('DesktopIncompleteError')).toBe(false); + }); }); describe('redactDesktopSecrets', () => { diff --git a/packages/agentic/desktop-test/src/reporting/protocol.ts b/packages/agentic/desktop-test/src/reporting/protocol.ts index 46bfa5b69f..fbdc1df6d9 100644 --- a/packages/agentic/desktop-test/src/reporting/protocol.ts +++ b/packages/agentic/desktop-test/src/reporting/protocol.ts @@ -16,6 +16,38 @@ export class DesktopReportedError extends Error { this.failure = failure; } } + +const DESKTOP_INCOMPLETE_ERROR = 'DesktopIncompleteError'; + +/** + * Selected work finished without producing complete executable coverage: a skipped or unsupported case, a group that + * reported no tests, or a producer that did not finish. It is a verdict, not a fault, so reporters record it as an + * `incomplete` run instead of an infrastructure failure. + */ +export class DesktopIncompleteError extends Error { + constructor(message: string) { + super(message); + this.name = DESKTOP_INCOMPLETE_ERROR; + } +} + +/** + * True for a {@link DesktopIncompleteError} thrown here or relayed by name over IPC, or an aggregate of only such errors. + * A different error that merely wraps one is a fault, so it is not recognized. + */ +export function isDesktopIncompleteError(error: unknown): boolean { + if (error instanceof DesktopIncompleteError) return true; + if (error instanceof DesktopReportedError) return isDesktopIncompleteFailure(error.failure); + if (error instanceof AggregateError && error.errors.length > 0) + return error.errors.every((child: unknown) => isDesktopIncompleteError(child)); + return false; +} + +/** True for a failure that is an incomplete-run verdict, which crosses a process boundary by error name. */ +export function isDesktopIncompleteFailure(failure: ReportFailure): boolean { + return failure.error.name === DESKTOP_INCOMPLETE_ERROR; +} + export type TestStatus = 'passed' | 'failed' | 'skipped' | 'unsupported' | 'cancelled' | 'not-run' | 'incomplete'; /** A declared gap: the producer knows the case cannot run here and says why, so it is not a silent skip. */ export type TestGap = 'unsupported' | 'quarantine'; @@ -129,7 +161,11 @@ export function createEventEmitter(runId: string, sourceId: string, receive: (ev }); } -export function validateDesktopEvent(value: unknown): asserts value is DesktopEvent { +/** Events only the collector may originate; a producer that sends one is misreporting the run's outcome. */ +export const collectorOnlyEventKinds: readonly DesktopEvent['kind'][] = ['run-finished']; + +/** `origin: 'collector'` admits {@link collectorOnlyEventKinds}; every other caller validates a producer's event. */ +export function validateDesktopEvent(value: unknown, origin: 'producer' | 'collector' = 'producer'): asserts value is DesktopEvent { const event = record(value, 'report event'); exactKeys(event, [ 'schemaVersion', @@ -152,6 +188,7 @@ export function validateDesktopEvent(value: unknown): asserts value is DesktopEv if (typeof event.kind !== 'string' || !Object.hasOwn(payloadFields, event.kind)) throw new TypeError('Unknown desktop report event kind.'); const kind = event.kind as keyof typeof payloadFields; + if (origin === 'producer' && collectorOnlyEventKinds.includes(kind)) throw new TypeError(`A producer cannot report "${kind}".`); const payload = record(event.payload, 'event payload'); exactKeys(payload, payloadFields[kind]); if (event.context !== undefined) { diff --git a/packages/agentic/desktop-test/src/reporting/results.test.ts b/packages/agentic/desktop-test/src/reporting/results.test.ts index 7953183e5d..0d160785e2 100644 --- a/packages/agentic/desktop-test/src/reporting/results.test.ts +++ b/packages/agentic/desktop-test/src/reporting/results.test.ts @@ -2,6 +2,9 @@ import os from 'node:os'; import path from 'node:path'; import { + decideDesktopTestStatus, + decisiveDesktopAttempt, + describeDesktopIncompleteRun, describeDesktopSelection, describeDesktopUnsupported, projectDesktopRunSummary, @@ -225,3 +228,109 @@ describe('bounded desktop run summary', () => { expect(summary.artifacts.runJson).toBe(path.join(root, 'run.json')); }); }); + +describe('the attempt that decides a test', () => { + const attempt = (status: DesktopTestResult['status'], index: number, reason = `${status} ${index}`) => ({ + attempt: index, + status, + durationMs: 1, + reason, + ...(status === 'failed' + ? { failure: { failureId: `failure-${index}`, kind: 'test' as const, error: { name: 'Error', message: reason } } } + : {}), + }); + + test.each([ + [['passed'], 'passed'], + [['failed', 'passed'], 'passed'], + [['failed', 'skipped'], 'failed'], + [['failed', 'unsupported'], 'failed'], + [['failed', 'incomplete'], 'failed'], + [['failed', 'cancelled'], 'failed'], + [['failed', 'skipped', 'passed'], 'passed'], + [['skipped', 'unsupported'], 'unsupported'], + [['incomplete', 'skipped'], 'skipped'], + [[], 'incomplete'], + ] as const)('attempts %j decide %s', (statuses, expected) => { + expect(decideDesktopTestStatus(statuses.map((status) => ({ status })))).toBe(expected); + }); + + test('reports a failed-then-skipped test with the failed attempt in every rendering', () => { + const test: DesktopTestResult = { + testId: 'masked', + title: 'masked', + status: 'failed', + flaky: false, + attempts: [attempt('failed', 0, 'first attempt broke'), attempt('skipped', 1, 'retry was skipped')], + }; + expect(decisiveDesktopAttempt(test.attempts)).toBe(test.attempts[0]); + const result = summarize([outcome('passed'), test]); + expect(result.status).toBe('failed'); + expect(renderJUnit(result)).toContain(''); + expect(projectDesktopRunSummary(result, '/run')).toMatchObject({ + problems: [{ kind: 'test', id: 'masked', status: 'failed', message: 'first attempt broke' }], + }); + }); + + test('a declared gap is read from the attempt that decided the test', () => { + const test: DesktopTestResult = { + testId: 'gap', + title: 'gap', + status: 'unsupported', + flaky: false, + attempts: [ + { ...attempt('skipped', 0), status: 'skipped' }, + { ...attempt('unsupported', 1), gap: 'unsupported' }, + ], + }; + expect(summarize([outcome('passed'), test], { lane: accessible }).counts).toMatchObject({ declaredUnsupported: 1 }); + }); +}); + +describe('incomplete runs explain themselves', () => { + test('a lane run names each skipped and undeclared case and gives JUnit the real reason', () => { + const result = summarize([outcome('passed', undefined, 'ran'), outcome('skipped', undefined, 'skipped-case'), outcome('unsupported')], { + lane: accessible, + }); + expect(result.status).toBe('incomplete'); + const summary = projectDesktopRunSummary(result, '/run'); + expect(summary.problems.map(({ id, status }) => [id, status])).toEqual([ + ['skipped-case', 'skipped'], + ['unsupported-plain', 'unsupported'], + ]); + expect(summary.problems[0].message).toContain('Not executed.'); + const xml = renderJUnit(result); + expect(xml).toContain('2 selected tests did not execute in the accessible lane'); + expect(xml).not.toContain('No required tests executed or a producer did not finish.'); + expect(renderRunSummary(result)).toContain('Desktop run is incomplete; 2 selected tests did not execute in the accessible lane'); + }); + + test('a run that reported nothing says so rather than blaming a producer', () => { + const result = summarize([], { lane: accessible }); + expect(projectDesktopRunSummary(result, '/run').problems).toEqual([ + expect.objectContaining({ kind: 'unfinished', id: 'run', status: 'incomplete', message: 'No tests were reported.' }), + ]); + expect(renderJUnit(result)).toContain('message="No tests were reported."'); + }); + + test('unfinished work carries the reason its producer was overruled', () => { + const result = summarizeDesktopRun({ + schemaVersion: 1, + runId: 'run-id', + kind: 'tests', + startedAt: new Date(0).toISOString(), + finishedAt: new Date(1).toISOString(), + durationMs: 1, + tests: [outcome('passed')], + work: [{ workId: 'work', label: 'empty spec', status: 'incomplete', reason: 'No tests were reported for this work.' }], + failures: [], + artifacts: [], + }); + expect(result.status).toBe('incomplete'); + expect(projectDesktopRunSummary(result, '/run').problems).toEqual([ + expect.objectContaining({ kind: 'unfinished', id: 'work', message: 'No tests were reported for this work.' }), + ]); + expect(renderJUnit(result)).toContain('message="No tests were reported for this work."'); + expect(describeDesktopIncompleteRun(result)).toBe('Desktop run is incomplete; empty spec: No tests were reported for this work.'); + }); +}); diff --git a/packages/agentic/desktop-test/src/reporting/results.ts b/packages/agentic/desktop-test/src/reporting/results.ts index 684a0d6ffc..369fa5b00f 100644 --- a/packages/agentic/desktop-test/src/reporting/results.ts +++ b/packages/agentic/desktop-test/src/reporting/results.ts @@ -39,6 +39,8 @@ export type DesktopWorkResult = { workId: string; label: string; status: TestStatus; + /** Why the collector reports this work as `incomplete` although its producer finished, such as a group that reported no tests. */ + reason?: string; context?: EventContext; }; export type DesktopRunResult = { @@ -111,6 +113,75 @@ export type DesktopRunSummary = { const SUMMARY_MAX_BYTES = 4 * 1024; const SUMMARY_MAX_PROBLEMS = 10; +/** + * The status a test reports across its attempts, in the order they arrived. A passing final attempt passes the test, and + * it is flaky when an earlier attempt failed. Otherwise any failed attempt decides: a retry that was skipped, unsupported, + * or never finished cannot erase a failure. With no failed attempt the last attempt decides. + */ +export function decideDesktopTestStatus(attempts: readonly Pick[]): TestStatus { + const last = attempts.at(-1); + if (!last) return 'incomplete'; + if (last.status === 'passed') return 'passed'; + return attempts.some(({ status }) => status === 'failed') ? 'failed' : last.status; +} + +/** The latest attempt whose status is the test's decided status: the one a failure message or reason must come from. */ +export function decisiveDesktopAttempt>(attempts: readonly T[]): T | undefined { + const status = decideDesktopTestStatus(attempts); + for (let index = attempts.length - 1; index >= 0; index--) if (attempts[index].status === status) return attempts[index]; + return undefined; +} + +const unfinishedStatuses: readonly TestStatus[] = ['not-run', 'incomplete', 'cancelled']; + +/** + * Selected cases that stopped an `incomplete` run from passing although none failed: every skipped case and every + * unsupported one nothing declared in a lane run, or all of them when nothing executed. + */ +function unmetTests(result: Pick): DesktopTestResult[] { + if (result.status !== 'incomplete' || result.kind !== 'tests') return []; + const nothingExecuted = result.counts.executed === 0; + if (!nothingExecuted && result.lane === undefined) return []; + return result.tests.filter( + (test) => + test.status === 'skipped' || + (test.status === 'unsupported' && (nothingExecuted || decisiveDesktopAttempt(test.attempts)?.gap === undefined)), + ); +} + +/** + * Why an `incomplete` run is incomplete when no failed, unfinished, or unmet item says so on its own: nothing was + * reported, a lane left cases unexecuted, or nothing executed. + */ +function incompleteRunReason(result: Pick): string | undefined { + if ( + result.status !== 'incomplete' || + result.counts.incomplete > 0 || + result.work.some(({ status }) => unfinishedStatuses.includes(status)) + ) + return undefined; + if (result.kind === 'tests' && result.counts.selected === 0) return 'No tests were reported.'; + const unmet = result.counts.skipped + undeclaredUnsupported(result.counts); + if (result.lane && unmet > 0) + return `${unmet} selected ${unmet === 1 ? 'test' : 'tests'} did not execute in the ${result.lane.input} lane: skipped, or unsupported without a declaration.`; + if (result.kind === 'tests' && result.counts.executed === 0) return 'No required tests executed.'; + return 'A producer did not finish.'; +} + +/** One line saying why the run is not passing coverage, for the error that a selected-work check throws. */ +export function describeDesktopIncompleteRun(result: DesktopRunResult): string { + const parts = [ + ...result.work + .filter(({ status }) => unfinishedStatuses.includes(status)) + .slice(0, 3) + .map((work) => `${work.label}: ${work.reason ?? `work is ${work.status}`}`), + ...(result.counts.incomplete > 0 ? [`${result.counts.incomplete} tests did not finish`] : []), + ...(incompleteRunReason(result) ? [incompleteRunReason(result)!] : []), + ]; + const detail = parts.join('; ') || 'selected work did not produce complete executable results'; + return `Desktop run is ${result.status}; ${detail}${detail.endsWith('.') ? '' : '.'}`; +} + export function projectDesktopRunSummary(result: DesktopRunResult, root: string): DesktopRunSummary { const problems: DesktopRunSummary['problems'] = [ ...result.failures @@ -126,26 +197,48 @@ export function projectDesktopRunSummary(result: DesktopRunResult, root: string) ...result.tests .filter(({ status }) => status === 'failed' || ['cancelled', 'not-run', 'incomplete'].includes(status)) .map((test) => { - const latest = test.attempts.at(-1); + const decisive = decisiveDesktopAttempt(test.attempts); return { kind: 'test' as const, id: clipText(test.testId, 96), title: clipText(test.title, 128), status: test.status, - ...(latest?.failure ? { failureId: clipText(latest.failure.failureId, 96) } : {}), - ...(latest?.failure?.error.message || latest?.reason - ? { message: clipText(latest.failure?.error.message ?? latest.reason!, 256) } + ...(decisive?.failure ? { failureId: clipText(decisive.failure.failureId, 96) } : {}), + ...(decisive?.failure?.error.message || decisive?.reason + ? { message: clipText(decisive.failure?.error.message ?? decisive.reason!, 256) } : {}), }; }), + ...unmetTests(result).map((test) => { + const decisive = decisiveDesktopAttempt(test.attempts); + return { + kind: 'test' as const, + id: clipText(test.testId, 96), + title: clipText(test.title, 128), + status: test.status, + ...(decisive?.reason ? { message: clipText(describeReason(decisive), 256) } : {}), + }; + }), ...result.work - .filter(({ status }) => ['not-run', 'incomplete', 'cancelled'].includes(status)) + .filter(({ status }) => unfinishedStatuses.includes(status)) .map((work) => ({ kind: 'unfinished' as const, id: clipText(work.workId, 96), title: clipText(work.label, 128), status: work.status, + ...(work.reason ? { message: clipText(work.reason, 256) } : {}), })), + ...(incompleteRunReason(result) && unmetTests(result).length === 0 + ? [ + { + kind: 'unfinished' as const, + id: 'run', + title: 'Run', + status: result.status, + message: clipText(incompleteRunReason(result)!, 256), + }, + ] + : []), ]; const skippedReasons = [ ...new Set( @@ -252,13 +345,13 @@ export function summarizeDesktopRun( else if (test.status === 'skipped') counts.skipped++; else if (test.status === 'unsupported') { counts.unsupported++; - const gap = test.attempts.at(-1)?.gap; + const gap = decisiveDesktopAttempt(test.attempts)?.gap; if (gap === 'unsupported') counts.declaredUnsupported++; else if (gap === 'quarantine') counts.quarantined++; } } const infrastructureFailed = input.failures.some(({ kind }) => kind !== 'test') || input.work.some(({ status }) => status === 'failed'); - const unfinishedWork = input.work.some(({ status }) => ['not-run', 'incomplete', 'cancelled'].includes(status)); + const unfinishedWork = input.work.some(({ status }) => unfinishedStatuses.includes(status)); const noExecution = input.kind === 'tests' && counts.executed === 0; // A lane run promised to execute its whole selection. Only a declared gap may leave a selected case unexecuted, and a // run that executed nothing is never optional. @@ -328,20 +421,21 @@ export function desktopRunExitCode(result: DesktopRunResult): number { export function renderJUnit(result: DesktopRunResult): string { const infrastructure = result.failures.filter(({ kind }) => kind !== 'test'); - const unfinishedWork = result.work.filter(({ status }) => ['not-run', 'incomplete', 'cancelled'].includes(status)); - const incomplete = result.status === 'incomplete' && result.counts.incomplete === 0; + const unfinishedWork = result.work.filter(({ status }) => unfinishedStatuses.includes(status)); + const incompleteReason = incompleteRunReason(result); const cases = result.tests.map((test) => { const latest = test.attempts[test.attempts.length - 1]; + const decisive = decisiveDesktopAttempt(test.attempts) ?? latest; const history = JSON.stringify(test.attempts); const body = test.status === 'failed' - ? `${xml(JSON.stringify(latest.failure))}` + ? `${xml(JSON.stringify(decisive.failure))}` : test.status === 'passed' ? '' : ['skipped', 'unsupported'].includes(test.status) - ? `` - : ``; - return `${body}${xml(history)}`; + ? `` + : ``; + return `${body}${xml(history)}`; }); for (const failure of infrastructure) { cases.push( @@ -350,11 +444,11 @@ export function renderJUnit(result: DesktopRunResult): string { } for (const work of unfinishedWork) cases.push( - ``, + ``, ); - if (incomplete) + if (incompleteReason !== undefined) cases.push( - '', + ``, ); const properties: [string, string][] = []; if (result.lane) @@ -370,7 +464,7 @@ export function renderJUnit(result: DesktopRunResult): string { const propertyXml = properties.length ? `${properties.map(([name, value]) => ``).join('')}` : ''; - return `\n${propertyXml}${cases.join('')}\n`; + return `\n${propertyXml}${cases.join('')}\n`; } export function renderRunSummary(result: DesktopRunResult): string { @@ -381,6 +475,7 @@ export function renderRunSummary(result: DesktopRunResult): string { const selection = describeDesktopSelection(result); return ( `## Desktop tests: ${result.status}\n\n` + + (result.status === 'incomplete' ? `${markdown(describeDesktopIncompleteRun(result))}\n\n` : '') + `Run: \`${result.runId}\`; endpoint: ${result.context?.endpoint ?? 'native'}; renderer: ${result.context?.renderer ?? 'not applicable'}.\n\n` + (selection ? `Selection: ${selection}.\n\n` : '') + `Selected ${c.selected}; executed ${c.executed}; passed ${c.passed}; failed ${c.failed}; skipped ${c.skipped}; ${describeDesktopUnsupported(c)}; incomplete ${c.incomplete}; retries ${c.retries}; flaky ${c.flaky}.\n\n` + diff --git a/packages/agentic/desktop-test/src/reporting/runQueries.test.ts b/packages/agentic/desktop-test/src/reporting/runQueries.test.ts index 6933f3b2f3..74a010a88d 100644 --- a/packages/agentic/desktop-test/src/reporting/runQueries.test.ts +++ b/packages/agentic/desktop-test/src/reporting/runQueries.test.ts @@ -4,7 +4,7 @@ import path from 'node:path'; import { randomUUID } from 'node:crypto'; import { DesktopRunReporter } from './collector.js'; -import { DesktopRunQueryService, validateDesktopRunQuery } from './runQueries.js'; +import { DesktopRunQueryError, DesktopRunQueryService, desktopRunQueryErrorCodes, validateDesktopRunQuery } from './runQueries.js'; import type { DesktopRunPage, DesktopRunQueryClient } from './runQueries.js'; import type { DesktopRunResult } from './results.js'; @@ -288,7 +288,7 @@ describe('persisted desktop run queries', () => { const retained = path.join(root, 'retained.json'); fs.renameSync(file, retained); fs.symlinkSync(retained, file); - await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toThrow('symlink'); + await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toMatchObject({ code: 'report-unreadable' }); fs.unlinkSync(file); fs.renameSync(retained, file); }, @@ -300,10 +300,10 @@ describe('persisted desktop run queries', () => { const file = path.join(reporter.root, 'run.json'); const linked = path.join(root, 'linked.json'); fs.linkSync(file, linked); - await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toThrow('private regular file'); + await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toMatchObject({ code: 'report-unreadable' }); fs.unlinkSync(linked); fs.truncateSync(file, 64 * 1024 * 1024 + 1); - await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toThrow('limit'); + await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toMatchObject({ code: 'report-unreadable' }); }); test('artifact limits, invalid UTF-8, and missing content fail explicitly', async () => { @@ -312,12 +312,57 @@ describe('persisted desktop run queries', () => { await reporter.flush(); const page = (await client.query({ operation: 'artifacts', runId: reporter.runId })) as DesktopRunPage; const query = { operation: 'artifact' as const, runId: reporter.runId, artifactId: page.items[0].id }; - await expect(client.query(query)).rejects.toThrow(); + await expect(client.query(query)).rejects.toMatchObject({ code: 'report-unreadable' }); const file = path.join(reporter.root, 'commands/output.log'); fs.writeFileSync(file, Buffer.from([0xff])); - await expect(client.query(query)).rejects.toThrow('encoded data'); + await expect(client.query(query)).rejects.toMatchObject({ code: 'report-invalid' }); fs.truncateSync(file, 1024 * 1024 + 1); - await expect(client.query(query)).rejects.toThrow('limit'); + await expect(client.query(query)).rejects.toMatchObject({ code: 'report-unreadable' }); + }); + + test('query errors carry a closed code and never expose a filesystem path', async () => { + testResult('passed'); + await reporter.finish(); + const file = path.join(reporter.root, 'run.json'); + const status = { operation: 'status' as const, runId: reporter.runId }; + const failures: unknown[] = []; + const attempt = async (query: Parameters[0]) => { + try { + await client.query(query); + } catch (error) { + failures.push(error); + } + }; + fs.writeFileSync(file, '{'); + await attempt(status); + fs.rmSync(file); + await attempt(status); + fs.rmSync(reporter.root, { force: true, recursive: true }); + await attempt(status); + await attempt({ operation: 'tests', runId: reporter.runId, cursor: 'not-a-cursor' }); + expect(failures).toHaveLength(4); + for (const failure of failures) { + expect(failure).toBeInstanceOf(DesktopRunQueryError); + expect(desktopRunQueryErrorCodes).toContain((failure as DesktopRunQueryError).code); + expect((failure as Error).message).not.toContain(root); + expect((failure as Error).message).not.toContain(path.basename(root)); + } + expect(failures.map((failure) => (failure as DesktopRunQueryError).code)).toEqual([ + 'report-invalid', + 'report-unreadable', + 'unavailable', + 'invalid-request', + ]); + }); + + test('projected text replaces the run directory instead of exposing it', async () => { + reporter.fail( + new Error(`Cannot write ${path.join(reporter.root, 'commands', 'output.log')} under ${path.dirname(reporter.root)}`), + 'test', + ); + await reporter.finish(); + const page = (await client.query({ operation: 'failures', runId: reporter.runId })) as DesktopRunPage; + expect(page.items[0].message).toBe('Cannot write /commands/output.log under '); }); test('malformed/inconsistent persisted data never returns a success-shaped fallback', async () => { @@ -327,9 +372,31 @@ describe('persisted desktop run queries', () => { result.tests[0].status = 'failed'; }); - await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toThrow('disagrees'); + await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toMatchObject({ code: 'report-invalid' }); fs.writeFileSync(path.join(reporter.root, 'run.json'), '{'); - await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toThrow(); + await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toMatchObject({ code: 'report-invalid' }); + }); + + test('a test that failed and then only skipped is queried as failed with its failure message', async () => { + const id = randomUUID(); + reporter.emit({ + kind: 'test-result', + payload: { + testId: id, + title: 'masked', + status: 'failed', + attempt: 0, + durationMs: 1, + failure: reporter.failure(new Error('first attempt broke'), 'test'), + }, + }); + reporter.emit({ + kind: 'test-result', + payload: { testId: id, title: 'masked', status: 'skipped', attempt: 1, durationMs: 1, reason: 'retry skipped' }, + }); + await reporter.finish(); + const tests = (await client.query({ operation: 'tests', runId: reporter.runId })) as DesktopRunPage; + expect(tests.items).toEqual([expect.objectContaining({ title: 'masked', status: 'failed', message: 'first attempt broke' })]); }); describe('input lane runs', () => { diff --git a/packages/agentic/desktop-test/src/reporting/runQueries.ts b/packages/agentic/desktop-test/src/reporting/runQueries.ts index 12a5f8d679..b42e18b7f2 100644 --- a/packages/agentic/desktop-test/src/reporting/runQueries.ts +++ b/packages/agentic/desktop-test/src/reporting/runQueries.ts @@ -1,8 +1,6 @@ import fs from 'node:fs'; import path from 'node:path'; import { createHash, randomUUID } from 'node:crypto'; -import { isNativeError } from 'node:util/types'; - import { desktopArtifactPath } from './collector.js'; import { DesktopReportedError, @@ -13,7 +11,7 @@ import { validateTestInputReport, } from './protocol.js'; import type { SelectionExclusions, TestGap, TestInputReport, TestStatus } from './protocol.js'; -import { desktopInputLaneSources, summarizeDesktopRun } from './results.js'; +import { decideDesktopTestStatus, decisiveDesktopAttempt, desktopInputLaneSources, summarizeDesktopRun } from './results.js'; import type { DesktopInputLaneReport, DesktopRunResult, DesktopSelectionReport } from './results.js'; export type DesktopRunQuery = @@ -73,6 +71,9 @@ type RegisteredRun = DesktopRunRegistration & { device: number; inode: number; artifacts: Map; + /** Every spelling of the run directory, and of its parent, that a producer's text can contain. Longest first. */ + roots: readonly string[]; + parents: readonly string[]; }; type Grant = DesktopRunReadGrant & { cursors: Map; @@ -83,7 +84,41 @@ const maxArtifactBytes = 1024 * 1024; const maxResponseBytes = 16 * 1024; const maxTextBytes = 1024; const maxCursors = 256; -const unavailable = () => new Error('Desktop run query is unavailable or unauthorized.'); + +/** + * The only failures a query can report. Each has a fixed message: a native error, which can carry an absolute path or an + * operating-system detail, never reaches a reader. + */ +export const desktopRunQueryErrorCodes = [ + 'unavailable', + 'invalid-request', + 'stale-cursor', + 'report-unreadable', + 'report-changed', + 'report-invalid', + 'internal', +] as const; +export type DesktopRunQueryErrorCode = (typeof desktopRunQueryErrorCodes)[number]; +const queryErrorMessages: Record, string> = { + unavailable: 'Desktop run query is unavailable or unauthorized.', + 'stale-cursor': 'Desktop run cursor is stale or unauthorized; request a new first page.', + 'report-unreadable': 'Retained desktop report is missing, not a private regular file, or exceeds its read limit.', + 'report-changed': 'Retained desktop report changed during reading; retry the query.', + 'report-invalid': 'Retained desktop report is not a valid desktop run: its encoded data or contents are invalid.', + internal: 'Desktop run query failed.', +}; + +export class DesktopRunQueryError extends Error { + readonly code: DesktopRunQueryErrorCode; + + /** `message` is accepted only for `invalid-request`, whose text comes from the request validator and names no path. */ + constructor(code: DesktopRunQueryErrorCode, message?: string) { + super(code === 'invalid-request' ? (message ?? 'Invalid desktop run query.') : queryErrorMessages[code]); + this.name = 'DesktopRunQueryError'; + this.code = code; + } +} +const unavailable = () => new DesktopRunQueryError('unavailable'); /** * Trusted host composition owns registrations and credentials. Readers cannot start work, discover roots, @@ -116,7 +151,17 @@ export class DesktopRunQueryService { throw new Error('Text artifact paths must be canonical relative paths other than run.json.'); artifacts.set(randomUUID(), relative); } - this.runs.set(registration.runId, { ...registration, root, device: stat.dev, inode: stat.ino, artifacts }); + const spellings = (candidates: readonly string[]) => + [...new Set(candidates)].filter((candidate) => candidate.length > 1).sort((a, b) => b.length - a.length); + this.runs.set(registration.runId, { + ...registration, + root, + device: stat.dev, + inode: stat.ino, + artifacts, + roots: spellings([root, registration.root, path.resolve(registration.root)]), + parents: spellings([path.dirname(root), path.dirname(registration.root), path.dirname(path.resolve(registration.root))]), + }); } forgetRun(runId: string): void { @@ -149,17 +194,26 @@ export class DesktopRunQueryService { try { return this.read(token, input); } catch (error) { - throw new Error(this.redact(isNativeError(error) ? error.message : 'Desktop run query failed.')); + throw error instanceof DesktopRunQueryError ? error : new DesktopRunQueryError('internal'); } } private read(token: string, input: unknown): DesktopRunQueryResponse { - validateDesktopRunQuery(input); + try { + validateDesktopRunQuery(input); + } catch (error) { + throw new DesktopRunQueryError('invalid-request', error instanceof TypeError ? error.message : undefined); + } const grant = this.grants.get(token); const run = this.runs.get(input.runId); if (!grant || !run || !grant.runIds.includes(input.runId) || grant.expiresAt <= Date.now() || run.expiresAt <= Date.now()) throw unavailable(); - const rootStat = fs.lstatSync(run.root); + let rootStat: fs.Stats; + try { + rootStat = fs.lstatSync(run.root); + } catch { + throw unavailable(); + } if (!rootStat.isDirectory() || rootStat.dev !== run.device || rootStat.ino !== run.inode) throw unavailable(); if ((input.operation === 'artifact' || input.operation === 'artifacts') && !grant.artifacts) throw unavailable(); const retained = readRetainedText(run.root, 'run.json', maxResultBytes); @@ -168,7 +222,7 @@ export class DesktopRunQueryService { if (input.operation === 'artifact') { const relative = run.artifacts.get(input.artifactId); if (!relative || !result.artifacts.some((artifact) => artifact.path === relative)) throw unavailable(); - const text = this.redact(readRetainedText(run.root, relative, maxArtifactBytes)); + const text = this.present(run, readRetainedText(run.root, relative, maxArtifactBytes)); const clipped = clip(text, 4 * 1024); return { schemaVersion: 1, @@ -200,7 +254,7 @@ export class DesktopRunQueryService { if (input.cursor) { const cursor = grant.cursors.get(input.cursor); if (!cursor || cursor.runId !== run.runId || cursor.operation !== input.operation || cursor.digest !== digest) - throw new Error('Desktop run cursor is stale or unauthorized; request a new first page.'); + throw new DesktopRunQueryError('stale-cursor'); offset = cursor.offset; } const items = this.items(input.operation, canonical, run); @@ -214,7 +268,7 @@ export class DesktopRunQueryService { omissions: { items: items.length - offset, textBytes: 0 }, }; for (const item of items.slice(offset, offset + (input.limit ?? 10))) { - const clipped = this.clipItem(item); + const clipped = this.clipItem(item, run); page.items.push(clipped.item); if (Buffer.byteLength(JSON.stringify(page)) > maxResponseBytes - 128) { page.items.pop(); @@ -236,9 +290,17 @@ export class DesktopRunQueryService { return redactDesktopSecrets(text, this.secrets); } - private clipItem(item: DesktopRunQueryItem): { item: DesktopRunQueryItem; omitted: number } { - const title = clip(this.redact(item.title), maxTextBytes); - const message = item.message === undefined ? undefined : clip(this.redact(item.message), maxTextBytes); + // Projected text is for a reader that must not learn where the run lives, so the run's own directories become placeholders. + private present(run: RegisteredRun, text: string): string { + let result = this.redact(text); + for (const root of run.roots) result = result.split(root).join(''); + for (const parent of run.parents) result = result.split(parent).join(''); + return result; + } + + private clipItem(item: DesktopRunQueryItem, run: RegisteredRun): { item: DesktopRunQueryItem; omitted: number } { + const title = clip(this.present(run, item.title), maxTextBytes); + const message = item.message === undefined ? undefined : clip(this.present(run, item.message), maxTextBytes); return { item: { id: item.id, @@ -255,15 +317,18 @@ export class DesktopRunQueryService { const id = (category: string, value: string) => createHash('sha256').update(`${run.runId}:${category}:${value}`).digest('hex'); switch (operation) { case 'tests': - return result.tests.map((test) => ({ - id: id('test', test.testId), - title: test.title, - status: test.status, - ...(test.attempts.at(-1)?.gap ? { gap: test.attempts.at(-1)!.gap } : {}), - message: test.attempts.at(-1)?.failure?.error.message ?? test.attempts.at(-1)?.reason, - })); + return result.tests.map((test) => { + const decisive = decisiveDesktopAttempt(test.attempts); + return { + id: id('test', test.testId), + title: test.title, + status: test.status, + ...(decisive?.gap ? { gap: decisive.gap } : {}), + message: decisive?.failure?.error.message ?? decisive?.reason, + }; + }); case 'work': - return result.work.map((work) => ({ id: id('work', work.workId), title: work.label, status: work.status })); + return result.work.map((work) => ({ id: id('work', work.workId), title: work.label, status: work.status, message: work.reason })); case 'failures': return result.failures.map((failure) => ({ id: id('failure', failure.failureId), @@ -290,7 +355,11 @@ export class DesktopRunQueryClient { } async query(query: DesktopRunQuery): Promise { - validateDesktopRunQuery(query); + try { + validateDesktopRunQuery(query); + } catch (error) { + throw new DesktopRunQueryError('invalid-request', error instanceof TypeError ? error.message : undefined); + } return this.request(query); } } @@ -317,22 +386,41 @@ export function validateDesktopRunQuery(value: unknown): asserts value is Deskto } function readRetainedText(root: string, relative: string, maximum: number): string { - const file = desktopArtifactPath(root, relative); - const fd = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK); + let file: string; + let fd: number; + try { + file = desktopArtifactPath(root, relative); + fd = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK); + } catch { + throw new DesktopRunQueryError('report-unreadable'); + } try { const stat = fs.fstatSync(fd); - if (!stat.isFile() || stat.nlink !== 1 || stat.size > maximum) - throw new Error('Retained desktop report exceeds its read limit or is not a private regular file.'); + if (!stat.isFile() || stat.nlink !== 1 || stat.size > maximum) throw new DesktopRunQueryError('report-unreadable'); const buffer = Buffer.alloc(Math.min(stat.size + 1, maximum + 1)); const length = fs.readSync(fd, buffer, 0, buffer.length, 0); - if (length !== stat.size || length > maximum) throw new Error('Retained desktop report changed during reading; retry the query.'); - return new TextDecoder('utf-8', { fatal: true }).decode(buffer.subarray(0, length)); + if (length !== stat.size || length > maximum) throw new DesktopRunQueryError('report-changed'); + try { + return new TextDecoder('utf-8', { fatal: true }).decode(buffer.subarray(0, length)); + } catch { + throw new DesktopRunQueryError('report-invalid'); + } + } catch (error) { + throw error instanceof DesktopRunQueryError ? error : new DesktopRunQueryError('report-unreadable'); } finally { fs.closeSync(fd); } } function parseResult(text: string, runId: string): DesktopRunResult & { terminal?: boolean } { + try { + return parsePersistedResult(text, runId); + } catch { + throw new DesktopRunQueryError('report-invalid'); + } +} + +function parsePersistedResult(text: string, runId: string): DesktopRunResult & { terminal?: boolean } { let input: unknown; try { input = JSON.parse(text); @@ -386,7 +474,8 @@ function parseResult(text: string, runId: string): DesktopRunResult & { terminal ...(attempt.failure !== undefined ? { failure: new DesktopReportedError(attempt.failure).failure } : {}), }; }); - if (attempts.at(-1)!.status !== test.status) throw new TypeError('Persisted desktop test status disagrees with its latest attempt.'); + if (decideDesktopTestStatus(attempts) !== test.status) + throw new TypeError('Persisted desktop test status disagrees with the status its attempts decide.'); return { testId: test.testId, title: test.title, status: test.status, flaky: test.flaky, attempts }; }); const work = (value.work as unknown[]).map((entry) => { @@ -394,7 +483,13 @@ function parseResult(text: string, runId: string): DesktopRunResult & { terminal textField(item.workId); textField(item.label); statusField(item.status); - return { workId: item.workId, label: item.label, status: item.status }; + if (item.reason !== undefined) textField(item.reason); + return { + workId: item.workId, + label: item.label, + status: item.status, + ...(typeof item.reason === 'string' ? { reason: item.reason } : {}), + }; }); const failures = (value.failures as unknown[]).map((entry) => new DesktopReportedError(entry).failure); const artifacts = (value.artifacts as unknown[]).map((entry) => { diff --git a/packages/agentic/desktop-test/src/reporting/session.ts b/packages/agentic/desktop-test/src/reporting/session.ts index 78392e6f94..9f8cc14917 100644 --- a/packages/agentic/desktop-test/src/reporting/session.ts +++ b/packages/agentic/desktop-test/src/reporting/session.ts @@ -1,8 +1,9 @@ import { randomUUID } from 'node:crypto'; import { DesktopRunReporter, desktopArtifactPath } from './collector.js'; import type { DesktopReportWriter, DesktopRunReporterOptions } from './collector.js'; -import { DesktopReportedError, redactDesktopSecrets, sanitizeDesktopReport, serializeReportError } from './protocol.js'; -import type { EventEmitter, FailureKind, ReportFailure } from './protocol.js'; +import { redactDesktopSecrets, sanitizeDesktopReport } from './protocol.js'; +import type { EventEmitter, FailureKind } from './protocol.js'; +import { DesktopFailureRegistry } from './failures.js'; import { assertReportedWork, connectReportProducer, DESKTOP_REPORT_CONTEXT, startReportTransport } from './transport.js'; import { desktopRunExitCode } from './results.js'; import type { DesktopRunResult } from './results.js'; @@ -69,16 +70,17 @@ export async function createDesktopReportSession( // connectReportProducer has already validated this inherited context. const context = JSON.parse(encoded) as { runId: string; artifactsRoot: string; nonce: string }; const secrets = new Set([context.nonce]); - const failures = new WeakMap(); const redact = (text: string) => redactDesktopSecrets(text, secrets); const protect = (emit: EventEmitter): EventEmitter => (event, scope, context, parent) => emit(sanitizeDesktopReport(event, redact), scope, sanitizeDesktopReport(context ?? {}, redact), parent); + const emit = protect(producer.emit); + const registry = new DesktopFailureRegistry((value) => sanitizeDesktopReport(value, redact)); const reporter: DesktopReportWriter = { runId: context.runId, root: context.artifactsRoot, - emit: protect(producer.emit), + emit, source: (id) => protect(producer.source(id)), addSecret(secret) { if (secret) secrets.add(secret); @@ -94,33 +96,17 @@ export async function createDesktopReportSession( await assertReportedWork(workIds, optional, encoded); }, failure(error, kind: FailureKind = 'infrastructure') { - if (error instanceof DesktopReportedError) return sanitizeDesktopReport(error.failure, redact); - if (error instanceof Error && error.cause instanceof DesktopReportedError) return sanitizeDesktopReport(error.cause.failure, redact); - const object = error !== null && typeof error === 'object' ? error : undefined; - const prior = - (object && failures.get(object)) || - (error instanceof Error && error.cause && typeof error.cause === 'object' ? failures.get(error.cause) : undefined); - if (prior) return prior; - const failure = { - failureId: randomUUID(), - kind, - error: JSON.parse(redact(JSON.stringify(serializeReportError(error)))) as ReportFailure['error'], - }; - if (object) failures.set(object, failure); - return failure; + return registry.failure(error, kind); }, - fail(error, kind, scopeId) { - if (error instanceof AggregateError && error.errors.length) { - const children: unknown[] = error.errors; - return children.map((child) => reporter.fail(child, kind, scopeId))[0]; - } - if (error instanceof Error && error.cause instanceof AggregateError) return reporter.fail(error.cause, kind, scopeId); - const failure = reporter.failure(error, kind); - producer.emit( - { kind: 'diagnostic', payload: { level: 'error', message: failure.error.message || failure.error.name, failure } }, - scopeId, + fail(error, kind = 'infrastructure', scopeId) { + return registry.fail( + error, + kind, + (failure) => + emit({ kind: 'diagnostic', payload: { level: 'error', message: failure.error.message || failure.error.name, failure } }, scopeId), + // The collector, which sees the run's status, decides whether an incomplete-run error is a failure or a warning. + () => false, ); - return failure; }, }; return { diff --git a/packages/agentic/desktop-test/src/reporting/structured.test.ts b/packages/agentic/desktop-test/src/reporting/structured.test.ts index 23b29dd175..6786d90772 100644 --- a/packages/agentic/desktop-test/src/reporting/structured.test.ts +++ b/packages/agentic/desktop-test/src/reporting/structured.test.ts @@ -57,6 +57,49 @@ describe('structured desktop events', () => { expect(record.payload.text).toContain('[truncated]'); expect(Buffer.byteLength(JSON.stringify(record))).toBeLessThan(4 * 1024); }); + + test('keeps identifiers whole while shrinking free text to fit the record limit', () => { + const testId = 'test-'.repeat(120); + const event = makeEvent({ + kind: 'test-result', + payload: { + testId, + title: 'title '.repeat(2_000), + attempt: 0, + status: 'failed', + durationMs: 1, + failure: { failureId: 'f'.repeat(400), kind: 'test', error: { name: 'Error', message: 'message '.repeat(2_000) } }, + }, + }); + const record = projectDesktopEvent(event); + + expect(record.payload.testId).toBe(testId); + expect((record.payload.failure as { failureId: string }).failureId).toBe('f'.repeat(400)); + expect(record.truncated).toBeUndefined(); + expect(Buffer.byteLength(JSON.stringify(record))).toBeLessThanOrEqual(4 * 1024); + }); + + test('an event that cannot fit even with short text becomes a marked minimal record instead of throwing', () => { + const long = (prefix: string) => `${prefix}-`.padEnd(1_000, 'x'); + const event = makeEvent({ + kind: 'test-result', + payload: { + testId: long('test'), + title: 'title', + attempt: 2, + status: 'failed', + durationMs: 1, + failure: { failureId: long('failure'), kind: 'test', error: { name: 'Error', message: 'message' } }, + }, + }); + const record = { ...event, runId: long('run'), eventId: long('event'), sourceId: long('source'), scopeId: long('scope') }; + + const projected = projectDesktopEvent(record as CollectedDesktopEvent); + + expect(projected.truncated).toBe(true); + expect(projected).toMatchObject({ kind: 'test-result', payload: { attempt: 2, status: 'failed' } }); + expect(Buffer.byteLength(JSON.stringify(projected))).toBeLessThanOrEqual(4 * 1024); + }); }); function makeEvent(data: Pick): CollectedDesktopEvent { diff --git a/packages/agentic/desktop-test/src/reporting/structured.ts b/packages/agentic/desktop-test/src/reporting/structured.ts index ed3ded7ee0..79de70d1c3 100644 --- a/packages/agentic/desktop-test/src/reporting/structured.ts +++ b/packages/agentic/desktop-test/src/reporting/structured.ts @@ -15,10 +15,16 @@ export type DesktopEventRecord = { parentScopeId?: string; context?: DesktopEvent['context']; payload: Record; + /** Set when the record is the reduced form of an event that could not fit its size limit; the journal holds the whole event. */ + truncated?: true; }; const maximumRecordBytes = 4 * 1024; const maximumTextBytes = 300; +// Identifiers are how a consumer finds the event, test, or failure again, so only an absurd one is ever shortened. +const maximumIdBytes = 1024; +const fallbackIdBytes = 128; +const payloadIdFields = ['testId', 'workId']; export function includeDesktopEvent(event: DesktopEvent, level: DesktopEventLevel, scopeCategory?: string): boolean { if (level === 'all') return true; @@ -51,7 +57,7 @@ export function projectDesktopEvent(event: CollectedDesktopEvent): DesktopEventR const payload: Record = {}; switch (event.kind) { case 'work-selected': - payload.workId = clip(event.payload.workId); + payload.workId = id(event.payload.workId); payload.label = clip(event.payload.label); break; case 'scope-start': @@ -79,12 +85,12 @@ export function projectDesktopEvent(event: CollectedDesktopEvent): DesktopEventR payload.text = clip(event.payload.text); break; case 'test-start': - payload.testId = clip(event.payload.testId); + payload.testId = id(event.payload.testId); payload.title = clip(event.payload.title); payload.attempt = event.payload.attempt; break; case 'test-result': - payload.testId = clip(event.payload.testId); + payload.testId = id(event.payload.testId); payload.title = clip(event.payload.title); payload.attempt = event.payload.attempt; payload.status = event.payload.status; @@ -111,14 +117,14 @@ export function projectDesktopEvent(event: CollectedDesktopEvent): DesktopEventR const record: DesktopEventRecord = { schemaVersion: 1, type: 'event', - runId: clip(event.runId), - eventId: clip(event.eventId), - sourceId: clip(event.sourceId), + runId: id(event.runId), + eventId: id(event.eventId), + sourceId: id(event.sourceId), sourceSequence: event.sourceSequence, arrivalSequence: event.arrivalSequence, kind: event.kind, - scopeId: clip(event.scopeId), - ...(event.parentScopeId ? { parentScopeId: clip(event.parentScopeId) } : {}), + scopeId: id(event.scopeId), + ...(event.parentScopeId ? { parentScopeId: id(event.parentScopeId) } : {}), ...(event.context ? { context: { @@ -135,22 +141,83 @@ export function projectDesktopEvent(event: CollectedDesktopEvent): DesktopEventR : {}), payload, }; - while (Buffer.byteLength(JSON.stringify(record)) > maximumRecordBytes) { - const key = Object.keys(payload).find((field) => typeof payload[field] === 'string' && (payload[field] as string).length > 32); - if (!key) throw new Error('Desktop event projection cannot fit within the 4 KiB record limit.'); - payload[key] = clip(payload[key] as string, 32); + return fitRecord(record); +} + +function byteLength(value: unknown): number { + return Buffer.byteLength(JSON.stringify(value)); +} + +type TextSlot = { holder: Record; key: string }; + +// Every free-text field a record carries. Identifiers are excluded: they are never shrunk to make room. +function textSlots(record: DesktopEventRecord): TextSlot[] { + const slots: TextSlot[] = []; + const collect = (holder: Record | undefined, skip: readonly string[] = []) => { + if (!holder) return; + for (const key of Object.keys(holder)) if (typeof holder[key] === 'string' && !skip.includes(key)) slots.push({ holder, key }); + }; + collect(record.payload, [...payloadIdFields, 'failureId']); + collect(record.context as Record | undefined); + const failure = record.payload.failure as { error?: Record } | undefined; + collect(failure?.error); + return slots; +} + +// Shrinks free text, longest first, until the record fits; if the identifiers and enums alone cannot fit, returns a +// minimal record rather than throwing, so one oversized event can never drop the run's terminal summary. +function fitRecord(record: DesktopEventRecord): DesktopEventRecord { + const slots = textSlots(record).sort( + (a, b) => Buffer.byteLength(b.holder[b.key] as string) - Buffer.byteLength(a.holder[a.key] as string), + ); + for (const { holder, key } of slots) { + if (byteLength(record) <= maximumRecordBytes) return record; + if (Buffer.byteLength(holder[key] as string) > 32) holder[key] = clip(holder[key] as string, 32); } - return record; + if (byteLength(record) <= maximumRecordBytes) return record; + const payload: Record = {}; + for (const key of [...payloadIdFields, 'attempt', 'status', 'durationMs', 'level', 'category', 'stream', 'gap', 'excluded']) + if (record.payload[key] !== undefined) payload[key] = record.payload[key]; + const failure = record.payload.failure as { failureId: string; kind: string } | undefined; + if (failure) payload.failure = { failureId: failure.failureId, kind: failure.kind }; + const reduced: DesktopEventRecord = { + schemaVersion: 1, + type: 'event', + runId: record.runId, + eventId: record.eventId, + sourceId: record.sourceId, + sourceSequence: record.sourceSequence, + arrivalSequence: record.arrivalSequence, + kind: record.kind, + scopeId: record.scopeId, + ...(record.parentScopeId ? { parentScopeId: record.parentScopeId } : {}), + payload, + truncated: true, + }; + if (byteLength(reduced) <= maximumRecordBytes) return reduced; + const shorten = (value: string) => clip(value, fallbackIdBytes); + reduced.runId = shorten(reduced.runId); + reduced.eventId = shorten(reduced.eventId); + reduced.sourceId = shorten(reduced.sourceId); + reduced.scopeId = shorten(reduced.scopeId); + if (reduced.parentScopeId) reduced.parentScopeId = shorten(reduced.parentScopeId); + for (const key of payloadIdFields) if (typeof payload[key] === 'string') payload[key] = shorten(payload[key]); + if (failure) payload.failure = { failureId: shorten(failure.failureId), kind: failure.kind }; + return reduced; } function projectFailure(failure: { failureId: string; kind: string; error: { name: string; message: string } }) { return { - failureId: clip(failure.failureId), + failureId: id(failure.failureId), kind: failure.kind, error: { name: clip(failure.error.name), message: clip(failure.error.message) }, }; } +function id(value: string): string { + return clip(value, maximumIdBytes); +} + function clip(value: string, maximumBytes = maximumTextBytes): string { if (Buffer.byteLength(value) <= maximumBytes) return value; const marker = '[truncated]'; diff --git a/packages/agentic/desktop-test/src/reporting/transport.test.ts b/packages/agentic/desktop-test/src/reporting/transport.test.ts new file mode 100644 index 0000000000..882abb4cb2 --- /dev/null +++ b/packages/agentic/desktop-test/src/reporting/transport.test.ts @@ -0,0 +1,130 @@ +import fs from 'node:fs'; +import net from 'node:net'; +import os from 'node:os'; +import path from 'node:path'; +import { once } from 'node:events'; + +import { DesktopRunReporter } from './collector.js'; +import { createEventEmitter } from './protocol.js'; +import type { DesktopEvent } from './protocol.js'; +import { DESKTOP_REPORT_CONTEXT, requestReportStop, startReportTransport } from './transport.js'; +import type { ReportTransport, ReportTransportContext } from './transport.js'; + +const quiet = { write: () => true }; + +describe('desktop reporting transport rejects invalid producers', () => { + let root: string; + let reporter: DesktopRunReporter; + let transport: ReportTransport; + let context: ReportTransportContext; + let onFatal: jest.Mock; + beforeEach(async () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-transport-')); + reporter = new DesktopRunReporter({ artifactsRoot: root, kind: 'lifecycle', output: quiet, errorOutput: quiet, summary: false }); + onFatal = jest.fn(); + transport = await startReportTransport(reporter, onFatal); + context = JSON.parse(transport.environment[DESKTOP_REPORT_CONTEXT]); + }); + afterEach(async () => { + // One test already closed the transport. + await transport.close().catch(() => undefined); + fs.rmSync(root, { force: true, recursive: true }); + }); + + async function connect() { + const socket = net.createConnection(context.address); + socket.setEncoding('utf8'); + let received = ''; + socket.on('data', (chunk: string) => { + received += chunk; + }); + socket.on('error', () => undefined); + const closed = once(socket, 'close'); + await once(socket, 'connect'); + return { + socket, + send: (value: unknown) => socket.write(JSON.stringify(value) + '\n'), + received: () => received, + closed: async () => { + await closed; + return received; + }, + }; + } + + async function connectProducer(sourceId: string) { + const client = await connect(); + client.send({ ...context, type: 'hello', sourceId }); + await waitUntil(() => client.received().includes('"ready":true')); + return client; + } + + async function failures(): Promise { + return (await reporter.finish()).failures.map(({ error }) => error.message); + } + + test('a handshake with the wrong nonce', async () => { + const client = await connect(); + client.send({ ...context, nonce: 'not-the-nonce', type: 'hello', sourceId: 'producer' }); + expect(await client.closed()).toBe(''); + expect(await failures()).toEqual(['Invalid desktop reporting IPC handshake.']); + expect(onFatal).toHaveBeenCalledTimes(1); + }); + + test('a frame larger than 1 MiB', async () => { + const client = await connect(); + client.socket.write('x'.repeat(1024 * 1024 + 1)); + await client.closed(); + expect(await failures()).toEqual(['Desktop reporting IPC frame exceeded 1 MiB.']); + }); + + test.each([0, 1.5, 120_001])('a stop request with deadline %p', async (timeoutMs) => { + await expect(requestReportStop('scope', transport.environment[DESKTOP_REPORT_CONTEXT], timeoutMs)).rejects.toThrow(); + expect(await failures()).toEqual(['Invalid desktop stop deadline.']); + }); + + test('a result request with malformed or unauthenticated fields', async () => { + const malformed = await connect(); + malformed.send({ ...context, type: 'assert', workIds: 'work', optional: false }); + await malformed.closed(); + const unauthenticated = await connect(); + unauthenticated.send({ ...context, nonce: 'not-the-nonce', type: 'assert', workIds: [], optional: false }); + await unauthenticated.closed(); + expect(await failures()).toEqual(['Invalid selected-work result request.', 'Invalid desktop result request.']); + }); + + test('an event from another source after the handshake', async () => { + const client = await connectProducer('producer'); + const events: DesktopEvent[] = []; + createEventEmitter(context.runId, 'impostor', (event) => events.push(event))({ + kind: 'diagnostic', + payload: { level: 'warning', message: 'not mine' }, + }); + client.send(events[0]); + await client.closed(); + expect(await failures()).toEqual(['Reporting source changed after its handshake.']); + }); + + test('a producer that disconnects without finishing', async () => { + const client = await connectProducer('producer'); + client.socket.end(); + await client.closed(); + await waitUntil(() => onFatal.mock.calls.length > 0); + expect(await failures()).toEqual(['Reporting producer producer disconnected without its terminal result/flush handshake.']); + }); + + (process.platform === 'win32' ? test.skip : test)('close removes the private socket directory', async () => { + const directory = path.dirname(context.address); + expect(fs.statSync(directory).mode & 0o777).toBe(0o700); + await transport.close(); + expect(fs.existsSync(directory)).toBe(false); + }); +}); + +async function waitUntil(check: () => boolean): Promise { + const until = Date.now() + 5000; + while (!check()) { + if (Date.now() > until) throw new Error('Timed out waiting for the transport fixture.'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} diff --git a/packages/agentic/desktop-test/src/reporting/transport.ts b/packages/agentic/desktop-test/src/reporting/transport.ts index 771bbfa95a..a194190dc9 100644 --- a/packages/agentic/desktop-test/src/reporting/transport.ts +++ b/packages/agentic/desktop-test/src/reporting/transport.ts @@ -1,9 +1,8 @@ import net from 'node:net'; -import os from 'node:os'; -import path from 'node:path'; import { randomUUID, timingSafeEqual } from 'node:crypto'; import { once } from 'node:events'; +import { createLocalSocketAddress } from '../localSocket.js'; import { createEventEmitter, DesktopReportedError, validateDesktopEvent } from './protocol.js'; import type { DesktopEvent, EventEmitter } from './protocol.js'; import type { DesktopRunReporter } from './collector.js'; @@ -18,10 +17,8 @@ const maxFrame = 1024 * 1024; export async function startReportTransport(reporter: DesktopRunReporter, onFatal?: (error: Error) => void): Promise { const nonce = randomUUID(); reporter.addSecret(nonce); - const address = - process.platform === 'win32' - ? `\\\\.\\pipe\\furn-desktop-${reporter.runId}` - : path.join(os.tmpdir(), `furn-desktop-${reporter.runId}.sock`); + const endpoint = createLocalSocketAddress('desktop', reporter.runId); + const address = endpoint.address; const context: ReportTransportContext = { schemaVersion: 1, address, nonce, runId: reporter.runId, artifactsRoot: reporter.root }; const sockets = new Set(); const controls = new Map(); @@ -133,8 +130,13 @@ export async function startReportTransport(reporter: DesktopRunReporter, onFatal } }); }); - server.listen(address); - await once(server, 'listening'); + try { + server.listen(address); + await once(server, 'listening'); + } catch (error) { + endpoint.dispose(); + throw error; + } return { environment: { [DESKTOP_REPORT_CONTEXT]: JSON.stringify(context) }, async close() { @@ -147,6 +149,7 @@ export async function startReportTransport(reporter: DesktopRunReporter, onFatal await closed; } finally { clearTimeout(timer); + endpoint.dispose(); } }, }; diff --git a/packages/agentic/desktop-test/src/services.test.ts b/packages/agentic/desktop-test/src/services.test.ts new file mode 100644 index 0000000000..378af062bb --- /dev/null +++ b/packages/agentic/desktop-test/src/services.test.ts @@ -0,0 +1,123 @@ +import type { DesktopCommandRunner, PreparedDesktopCommand, RunningDesktopCommand } from './commands.js'; +import { DesktopServiceScope, DesktopStopRequested } from './services.js'; +import type { DesktopResourceContribution, DesktopResourceRecord } from './resources.js'; + +class FakeRunner implements DesktopCommandRunner { + readonly stopped: string[] = []; + readonly stopOptions: { failed?: boolean }[] = []; + async run(): Promise {} + start(command: PreparedDesktopCommand): RunningDesktopCommand { + let exit!: (code: number) => void; + const completed = new Promise((resolve) => { + exit = resolve; + }); + return { + completed, + stop: async (options = {}) => { + this.stopped.push(command.label!); + this.stopOptions.push(options); + exit(0); + }, + }; + } +} + +const context = { targetId: 'target', targetInstanceId: 'instance' }; +const command = { command: 'service', args: [], cwd: '.', env: {} }; + +function resource(id: string, disposed: string[]): DesktopResourceContribution { + return { + id, + ownership: 'owned', + prepare: () => ({ + identity: `fixture:${id}`, + acquire: async () => undefined, + dispose: async () => { + disposed.push(id); + }, + }), + }; +} + +describe('desktop service scope shutdown', () => { + test('disposes services and owned resources in reverse acquisition order', async () => { + const runner = new FakeRunner(); + const scope = new DesktopServiceScope(runner); + scope.start('first', command); + await scope.contribute([resource('middle', runner.stopped)], context); + scope.start('last', command); + await scope.close(); + expect(runner.stopped).toEqual(['last', 'middle', 'first']); + expect(runner.stopOptions).toEqual([{ failed: false }, { failed: false }]); + }); + + test('a service stopped after a scope failure is reported failed, and a stop request is not a failure', async () => { + const failedRunner = new FakeRunner(); + const failedScope = new DesktopServiceScope(failedRunner); + failedScope.start('service', command); + failedScope.fail(new Error('Required dependency failed')); + await failedScope.close(); + expect(failedRunner.stopOptions).toEqual([{ failed: true }]); + + const stoppedRunner = new FakeRunner(); + const stoppedScope = new DesktopServiceScope(stoppedRunner); + stoppedScope.start('service', command); + stoppedScope.requestStop(); + await stoppedScope.close(); + expect(stoppedRunner.stopOptions).toEqual([{ failed: false }]); + }); + + test('close settles work that waits for a stop request', async () => { + const scope = new DesktopServiceScope(new FakeRunner()); + const waiting = scope.guard(scope.waitForStop()); + await scope.close(); + await expect(waiting).rejects.toThrow('closing'); + }); + + test('closing from tracked work is bounded by the drain deadline', async () => { + const scope = new DesktopServiceScope(new FakeRunner(), undefined, 'desktop', { drainTimeoutMs: 100 }); + let inner: Promise | undefined; + const tracked = scope.guard( + (async () => { + await Promise.resolve(); + inner = scope.close(); + await inner; + })(), + ); + await expect(tracked).rejects.toThrow('closing'); + await expect(inner).rejects.toThrow('did not shut down cleanly'); + await expect(scope.close()).rejects.toMatchObject({ + errors: [expect.objectContaining({ message: expect.stringContaining('did not settle within 100ms') })], + }); + }); + + test.each([0, 1.5, -1, 2 ** 31])('rejects drainTimeoutMs %p', (drainTimeoutMs) => { + expect(() => new DesktopServiceScope(new FakeRunner(), undefined, 'desktop', { drainTimeoutMs })).toThrow(RangeError); + }); + + test('a resource observer failure during close still disposes everything and fails the close', async () => { + const runner = new FakeRunner(); + const scope = new DesktopServiceScope(runner); + const observed: DesktopResourceRecord[] = []; + scope.start('first', command); + scope.start('second', command); + scope.setResourceObserver((record) => { + observed.push(record); + if (record.disposal === 'pending' && observed.length > 2) throw new Error('Claim write failed'); + }); + scope.requestStop(); + await expect(scope.close()).rejects.toMatchObject({ errors: [expect.objectContaining({ message: 'Claim write failed' })] }); + expect(runner.stopped).toEqual(['second', 'first']); + }); + + test('a resource observer failure before close fails the scope', async () => { + const scope = new DesktopServiceScope(new FakeRunner()); + scope.setResourceObserver(() => { + throw new Error('Claim write failed'); + }); + expect(() => scope.start('service', command)).toThrow('Claim write failed'); + await expect(scope.waitForStop()).rejects.toThrow('Claim write failed'); + await expect(scope.close()).rejects.toThrow('did not shut down cleanly'); + expect(scope.controller.signal.reason).not.toBeInstanceOf(DesktopStopRequested); + }); +}); diff --git a/packages/agentic/desktop-test/src/services.ts b/packages/agentic/desktop-test/src/services.ts index af66045d7a..e0ab9904e0 100644 --- a/packages/agentic/desktop-test/src/services.ts +++ b/packages/agentic/desktop-test/src/services.ts @@ -25,6 +25,14 @@ type ResourceEntry = { disposalPromise?: Promise; }; +export type DesktopServiceScopeOptions = { + /** + * How long close() waits for tracked work (resource acquisitions and guarded operations) to settle after the scope is + * aborted, before each disposal pass. Work still running then is reported as a cleanup failure and abandoned. + */ + drainTimeoutMs?: number; +}; + export class DesktopStopRequested extends Error { constructor() { super('Owned desktop service stop requested.'); @@ -44,14 +52,26 @@ export class DesktopServiceScope { private closePromise?: Promise; private generation = 0; private onResource?: (resource: DesktopResourceRecord) => void; + private observerFailure?: Error; + private stopFailed = false; private readonly runner: DesktopCommandRunner; private readonly reporter?: DesktopReportWriter; private readonly resourcePrefix: string; + private readonly drainTimeoutMs: number; - constructor(runner: DesktopCommandRunner, reporter?: DesktopReportWriter, resourcePrefix = 'desktop') { + constructor( + runner: DesktopCommandRunner, + reporter?: DesktopReportWriter, + resourcePrefix = 'desktop', + options: DesktopServiceScopeOptions = {}, + ) { + const { drainTimeoutMs = 60_000 } = options; + if (!Number.isSafeInteger(drainTimeoutMs) || drainTimeoutMs <= 0 || drainTimeoutMs > 2_147_483_647) + throw new RangeError('Desktop service scope drainTimeoutMs must be a positive integer number of milliseconds.'); this.runner = runner; this.reporter = reporter; this.resourcePrefix = resourcePrefix; + this.drainTimeoutMs = drainTimeoutMs; this.failed = new Promise((_resolve, reject) => { this.rejectFailure = reject; }); @@ -61,7 +81,7 @@ export class DesktopServiceScope { setResourceObserver(observer?: (resource: DesktopResourceRecord) => void): void { this.onResource = observer; - if (observer) for (const resource of this.resources) observer(this.snapshot(resource)); + if (observer) for (const resource of this.resources) this.publish(resource); } fail(error: Error): void { @@ -91,13 +111,16 @@ export class DesktopServiceScope { }); let running: RunningDesktopCommand; try { + // A resource observer that failed while recording this service aborts the scope; do not start the process. + this.controller.signal.throwIfAborted(); + // Services are stopped in reverse acquisition order by close(), not by the scope signal. running = this.runner.start({ ...command, label: command.label ?? id, category: 'service', context: { ...command.context, service: id }, }); - resource.dispose = () => running.stop({ failed: true }); + resource.dispose = () => running.stop({ failed: this.stopFailed }); } catch (error) { resource.record.readiness = 'failed'; resource.record.disposal = 'settled'; @@ -255,30 +278,19 @@ export class DesktopServiceScope { ); } + /** + * Aborts the scope, waits (bounded) for tracked work, then disposes owned services and resources in reverse acquisition + * order. Never call it from work the scope tracks: close() waits for that work. + */ close(): Promise { return (this.closePromise ??= (async () => { this.closing = true; - const failed = this.controller.signal.aborted && !(this.controller.signal.reason instanceof DesktopStopRequested); - this.controller.abort(new Error('Desktop service scope is closing.')); + this.stopFailed = this.controller.signal.aborted && !(this.controller.signal.reason instanceof DesktopStopRequested); + const closingReason = new Error('Desktop service scope is closing.'); + this.controller.abort(closingReason); + // Work waiting for a stop request (waitForStop) settles too. + this.rejectFailure(closingReason); const failures: unknown[] = []; - for (const { id, command } of [...this.running].reverse()) { - try { - await deadline(command.stop({ failed }), 45_000, `Stopping ${id}`); - const resource = this.resources.find((entry) => entry.key === id); - if (resource) { - resource.record.disposal = 'settled'; - this.publish(resource); - } - } catch (error) { - failures.push(error); - this.reporter?.fail(error, 'cleanup', id); - const resource = this.resources.find((entry) => entry.key === id); - if (resource) { - resource.record.disposal = 'failed'; - this.publish(resource); - } - } - } await this.awaitWork(failures); for (const resource of [...this.resources].reverse()) { if (resource.record.ownership !== 'owned' || resource.record.disposal !== 'pending') continue; @@ -289,8 +301,8 @@ export class DesktopServiceScope { this.reporter?.fail(error, 'cleanup', resource.record.id); } } - await this.awaitWork(failures); + if (this.observerFailure) failures.push(this.observerFailure); if (failures.length) throw new AggregateError(failures, 'Owned desktop services did not shut down cleanly.'); })()); } @@ -309,8 +321,18 @@ export class DesktopServiceScope { return Object.freeze({ ...entry.record }); } + // The observer records ownership (the workload claim); a failure there must not interrupt acquisition or disposal. private publish(entry: ResourceEntry): void { - this.onResource?.(this.snapshot(entry)); + if (!this.onResource) return; + try { + this.onResource(this.snapshot(entry)); + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + if (this.observerFailure) return; + this.observerFailure = failure; + if (this.closing || this.controller.signal.aborted) this.reporter?.fail(failure, 'cleanup', entry.record.id); + else this.fail(failure); + } } private track(work: Promise): Promise { @@ -323,8 +345,21 @@ export class DesktopServiceScope { } private async awaitWork(failures: unknown[]): Promise { + const until = Date.now() + this.drainTimeoutMs; while (this.work.size > 0) { - const results = await Promise.allSettled(this.work); + let results: PromiseSettledResult[]; + try { + results = await deadline(Promise.allSettled(this.work), Math.max(0, until - Date.now()), 'Draining desktop scope work'); + } catch { + const error = new Error( + `${this.work.size} tracked desktop operation(s) did not settle within ${this.drainTimeoutMs}ms after the scope closed; abandoning them.`, + ); + // Abandoned work keeps its settlement handlers, so a late rejection is still observed. + this.work.clear(); + failures.push(error); + this.reporter?.fail(error, 'cleanup'); + return; + } for (const result of results) { if (result.status === 'rejected' && result.reason !== this.controller.signal.reason) failures.push(result.reason); } diff --git a/packages/agentic/desktop-test/src/signals.test.ts b/packages/agentic/desktop-test/src/signals.test.ts new file mode 100644 index 0000000000..1b5ca0090a --- /dev/null +++ b/packages/agentic/desktop-test/src/signals.test.ts @@ -0,0 +1,26 @@ +import { handleDesktopInterrupts } from './signals.js'; + +describe('desktop interrupt handling', () => { + test('escalates repeated interrupts and stays installed until released', () => { + const before = ['SIGINT', 'SIGTERM', 'SIGHUP'].map((signal) => process.listenerCount(signal)); + const stop = jest.fn(); + const terminate = jest.fn(); + const exit = jest.spyOn(process, 'exit').mockImplementation(() => undefined as never); + const release = handleDesktopInterrupts({ stop, terminate }); + try { + process.emit('SIGHUP', 'SIGHUP'); + expect(stop).toHaveBeenCalledWith('SIGHUP'); + process.emit('SIGINT', 'SIGINT'); + expect(terminate).toHaveBeenCalledWith('SIGINT'); + expect(exit).not.toHaveBeenCalled(); + process.emit('SIGTERM', 'SIGTERM'); + expect(exit).toHaveBeenCalledWith(143); + expect(stop).toHaveBeenCalledTimes(1); + expect(terminate).toHaveBeenCalledTimes(1); + } finally { + release(); + exit.mockRestore(); + } + expect(['SIGINT', 'SIGTERM', 'SIGHUP'].map((signal) => process.listenerCount(signal))).toEqual(before); + }); +}); diff --git a/packages/agentic/desktop-test/src/signals.ts b/packages/agentic/desktop-test/src/signals.ts new file mode 100644 index 0000000000..e394d41b10 --- /dev/null +++ b/packages/agentic/desktop-test/src/signals.ts @@ -0,0 +1,30 @@ +import os from 'node:os'; + +export type DesktopInterruptHandlers = { + /** First interrupt: request a graceful stop that disposes every owned process and resource. */ + stop(signal: NodeJS.Signals): void; + /** Second interrupt: force-kill owned process trees while cleanup continues. */ + terminate(signal: NodeJS.Signals): void; +}; + +const interruptSignals = ['SIGINT', 'SIGTERM', 'SIGHUP'] as const; + +/** + * Keeps interrupt handlers installed until the returned function is called, which must happen only after cleanup + * completes: a signal that arrives during cleanup must not fall back to Node's default and kill the process mid-disposal. + * A third interrupt exits immediately with the conventional 128 + signal status, leaving the workload claim behind for the + * next run to inspect. + */ +export function handleDesktopInterrupts(handlers: DesktopInterruptHandlers): () => void { + let received = 0; + const listener = (signal: NodeJS.Signals) => { + received++; + if (received === 1) handlers.stop(signal); + else if (received === 2) handlers.terminate(signal); + else process.exit(128 + (os.constants.signals[signal] ?? os.constants.signals.SIGINT)); + }; + for (const signal of interruptSignals) process.on(signal, listener); + return () => { + for (const signal of interruptSignals) process.off(signal, listener); + }; +} diff --git a/packages/agentic/desktop-test/src/supervisor.test.ts b/packages/agentic/desktop-test/src/supervisor.test.ts index 1489fc48c7..be170d3ab5 100644 --- a/packages/agentic/desktop-test/src/supervisor.test.ts +++ b/packages/agentic/desktop-test/src/supervisor.test.ts @@ -49,6 +49,8 @@ describe('desktop supervisor input lane', () => { target, reporter, reportSession: { reporter, delegated: true, environment: {}, close: async () => 0 }, + // Never contend for the real per-user workload claim. + lockDirectory: path.join(artifactsRoot, 'locks'), ...(input ? { input } : {}), }); try { diff --git a/packages/agentic/desktop-test/src/supervisor.ts b/packages/agentic/desktop-test/src/supervisor.ts index 6ac998ce8e..779f035973 100644 --- a/packages/agentic/desktop-test/src/supervisor.ts +++ b/packages/agentic/desktop-test/src/supervisor.ts @@ -240,7 +240,8 @@ export class DesktopSupervisor { } async worker(command: PreparedDesktopCommand, timeoutMs: number, verify?: (code: number) => Promise): Promise { - const worker = this.runner.start({ ...command, category: 'worker' }); + // An abort stops the worker's process tree at once instead of waiting for the guard to notice. + const worker = this.runner.start({ ...command, category: 'worker' }, { signal: this.services.controller.signal }); this.workers.add(worker); let failure: unknown; let code = 1; diff --git a/packages/agentic/desktop-test/src/wdio/configuration.ts b/packages/agentic/desktop-test/src/wdio/configuration.ts index b52c13597a..bac03c8fb8 100644 --- a/packages/agentic/desktop-test/src/wdio/configuration.ts +++ b/packages/agentic/desktop-test/src/wdio/configuration.ts @@ -72,6 +72,8 @@ export async function createDesktopWdioConfiguration(settings: DesktopWdioSettin bail: settings.bail, specFileRetries: settings.specFileRetries, logLevel: 'error', + // The framework adapters report a spec file that fails to import only as a warning and then run zero tests; keep that one visible. + logLevels: { '@wdio/jasmine-framework': 'warn', '@wdio/mocha-framework': 'warn' }, hostname: address.hostname, port: Number(address.port), path: address.pathname, diff --git a/packages/agentic/desktop-test/src/wdio/reporter.test.ts b/packages/agentic/desktop-test/src/wdio/reporter.test.ts new file mode 100644 index 0000000000..9d7c0ff2b6 --- /dev/null +++ b/packages/agentic/desktop-test/src/wdio/reporter.test.ts @@ -0,0 +1,132 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { PassThrough } from 'node:stream'; +import type { RunnerStats, TestStats } from '@wdio/reporter'; + +import { desktopJasmineRegistry } from '../jasmine/registry.js'; +import { DesktopRunReporter } from '../reporting/collector.js'; +import type { ReportProducer } from '../reporting/transport.js'; +import { connectReportProducer } from '../reporting/transport.js'; +import { wdioGroupId } from './identity.js'; +import DesktopWdioReporter from './reporter.js'; + +jest.mock('../reporting/transport', () => ({ connectReportProducer: jest.fn() })); + +type FixtureTest = { uid: string; title: string; state: 'passed' | 'failed' | 'pending' | 'skipped'; error?: Error }; + +describe('desktop WDIO reporter', () => { + const specs = ['/fixtures/example.spec.ts']; + const workId = wdioGroupId(specs); + const retries = 2; + let artifactsRoot: string; + let collector: DesktopRunReporter; + + beforeEach(() => { + artifactsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-wdio-reporter-')); + collector = new DesktopRunReporter({ artifactsRoot, output: { write: () => true }, reporterMode: 'json', summary: false }); + collector.emit({ kind: 'work-selected', payload: { workId, label: 'example spec' } }); + jest.mocked(connectReportProducer).mockImplementation(async (sourceId: string): Promise => { + const emit = collector.source(sourceId); + return { + emit, + source: (id) => collector.source(id), + flush: async () => {}, + finish: async (status = 'passed') => emit({ kind: 'producer-finished', payload: { status } }), + }; + }); + }); + + afterEach(() => { + const registry = desktopJasmineRegistry(); + registry.entries.clear(); + registry.blocked = false; + fs.rmSync(artifactsRoot, { force: true, recursive: true }); + }); + + /** One worker process: WDIO starts a fresh reporter for every attempt at the spec file. */ + async function runWorker(retry: number | undefined, tests: readonly FixtureTest[]): Promise { + const reporter = new DesktopWdioReporter({ + writeStream: new PassThrough(), + desktopContext: { endpoint: 'macos', renderer: 'fabric' }, + specFileRetries: retries, + } as ConstructorParameters[0]); + const failures = tests.filter(({ state }) => state === 'failed').length; + const runner = { cid: '0-0', specs, retry, failures } as unknown as RunnerStats; + reporter.onRunnerStart(runner); + for (const fixture of tests) { + const test = { ...fixture, fullTitle: `suite ${fixture.title}`, duration: 5, retries: 0, errors: [] } as unknown as TestStats; + reporter.onTestStart(test); + if (fixture.state === 'pending' || fixture.state === 'skipped') reporter.onTestSkip(test); + else reporter.onTestEnd(test); + } + reporter.onRunnerEnd(runner); + while (!reporter.isSynchronised) await new Promise((resolve) => setImmediate(resolve)); + } + + const flaky = { uid: 'test00-0', title: 'opens', state: 'failed', error: new Error('boom') } as const; + + test('orders attempts by arrival when WDIO reports N+1 for the first worker and 1..N for its retries', async () => { + await runWorker(retries + 1, [flaky]); + await runWorker(1, [flaky]); + await runWorker(2, [{ ...flaky, state: 'passed' }]); + const result = await collector.finish(); + expect(result.tests).toHaveLength(1); + expect(result.tests[0].attempts.map(({ status }) => status)).toEqual(['failed', 'failed', 'passed']); + const attempts = result.tests[0].attempts.map(({ attempt }) => attempt); + expect(new Set(attempts).size).toBe(3); + expect(attempts).toEqual([...attempts].sort((a, b) => a - b)); + expect(result.tests[0]).toMatchObject({ status: 'passed', flaky: true }); + expect(result.work).toMatchObject([{ workId, status: 'passed' }]); + }); + + test('a producer that numbers its first attempt from zero is still ordered and unique', async () => { + await runWorker(undefined, [flaky]); + await runWorker(1, [{ ...flaky, state: 'passed' }]); + const result = await collector.finish(); + expect(result.tests[0].attempts.map(({ attempt }) => attempt)).toEqual([0, 1_000_000]); + expect(result.tests[0].status).toBe('passed'); + }); + + test('a failed attempt followed only by a skipped retry keeps the test failed', async () => { + await runWorker(retries + 1, [flaky]); + await runWorker(1, [{ ...flaky, state: 'pending' }]); + const result = await collector.finish(); + expect(result.tests[0]).toMatchObject({ status: 'failed' }); + expect(result.status).toBe('failed'); + }); + + test('a spec that produced no tests is incomplete and says so', async () => { + await runWorker(1, []); + const result = await collector.finish(); + expect(result.status).toBe('incomplete'); + expect(result.work[0]).toMatchObject({ status: 'incomplete', reason: expect.stringContaining('No tests') }); + expect(fs.readFileSync(path.join(collector.root, 'events.jsonl'), 'utf8')).toContain('No tests ran for /fixtures/example.spec.ts'); + }); + + test('a spec whose cases were all excluded declares them and stays covered', async () => { + desktopJasmineRegistry().entries.set('suite hidden-0', { kind: 'excluded', reason: 'platform' }); + await runWorker(1, [{ uid: 'suite hidden-0', title: 'hidden', state: 'pending' }]); + const result = await collector.finish(); + expect(result.tests).toHaveLength(0); + expect(result.work[0].status).toBe('passed'); + expect(result.selection?.excluded).toMatchObject({ platform: 1 }); + expect(fs.readFileSync(path.join(collector.root, 'events.jsonl'), 'utf8')).not.toContain('No tests ran'); + }); + + test('maps framework states to desktop statuses', async () => { + await runWorker(1, [ + { uid: 'a-0', title: 'passes', state: 'passed' }, + { uid: 'b-0', title: 'fails', state: 'failed', error: new Error('nope') }, + { uid: 'c-0', title: 'pends', state: 'pending' }, + { uid: 'd-0', title: 'skips', state: 'skipped' }, + ]); + const result = await collector.finish(); + expect(Object.fromEntries(result.tests.map((test) => [test.title, test.status]))).toEqual({ + 'suite passes': 'passed', + 'suite fails': 'failed', + 'suite pends': 'skipped', + 'suite skips': 'skipped', + }); + }); +}); diff --git a/packages/agentic/desktop-test/src/wdio/reporter.ts b/packages/agentic/desktop-test/src/wdio/reporter.ts index bef914a414..dbba312216 100644 --- a/packages/agentic/desktop-test/src/wdio/reporter.ts +++ b/packages/agentic/desktop-test/src/wdio/reporter.ts @@ -59,8 +59,7 @@ export default class DesktopWdioReporter extends WDIOReporter { override onRunnerStart(runner: RunnerStats): void { this.groupId = wdioGroupId(runner.specs); - // WDIO reports the remaining spec attempts (including the current worker), not an increasing attempt index. - this.workerAttempt = this.specFileRetries + 1 - (runner.retry ?? 1); + this.workerAttempt = this.workerAttemptOf(runner.retry); this.scopeId = `${this.groupId}-${this.workerAttempt}`; this.context = { ...this.context, worker: runner.cid, spec: runner.specs.join(' + '), attempt: this.workerAttempt }; this.report({ @@ -168,14 +167,32 @@ export default class DesktopWdioReporter extends WDIOReporter { }, }); const excluded = excludedSpecCounts(desktopJasmineRegistry()); + const excludedCount = excluded.platform + excluded.scope + excluded.input; + // WDIO runs a spec that failed to import with zero tests and reports it as passed; the collector then marks this work incomplete. + if (!runner.error && this.endedTests.size === 0 && excludedCount === 0 && !desktopJasmineRegistry().blocked) + this.report({ + kind: 'diagnostic', + payload: { + level: 'warning', + message: `No tests ran for ${runner.specs.join(' + ')}; the spec file is empty or failed to load.`, + }, + }); // Keyed by the spec group, so a retried attempt that announces the same exclusions again is counted once. - if (excluded.platform + excluded.scope + excluded.input > 0) { + if (excludedCount > 0) { this.report({ kind: 'selection', payload: { excluded, key: this.groupId } }); } this.report({ kind: 'scope-end', payload: { status: runner.failures || runner.error ? 'failed' : 'passed' } }); this.schedule((producer) => producer.finish(runner.failures || runner.error ? 'failed' : 'passed')); } + /** + * The producer's attempt number is opaque: WDIO 10 reports `specFileRetries + 1` for the first worker and `1..specFileRetries` + * for its retries. The first worker is attempt 0 and a retry keeps its own number, so each worker attempt is unique and + * the collector orders attempts by arrival rather than by this number. + */ + private workerAttemptOf(retry: number | undefined): number { + return retry === undefined || !Number.isInteger(retry) || retry < 0 || retry > this.specFileRetries ? 0 : retry; + } private testId(test: TestStats): string { return createHash('sha256') .update(JSON.stringify([this.groupId, test.uid, test.fullTitle])) diff --git a/packages/agentic/desktop-test/src/wdio/run.ts b/packages/agentic/desktop-test/src/wdio/run.ts index e9db1b7bbf..43d452222f 100644 --- a/packages/agentic/desktop-test/src/wdio/run.ts +++ b/packages/agentic/desktop-test/src/wdio/run.ts @@ -10,6 +10,18 @@ import type { DesktopSupervisor } from '../supervisor.js'; import type { DesktopWdioSettings } from './configuration.js'; import { wdioGroupId } from './identity.js'; +const desktopWdioSettingsEnvironment = 'DESKTOP_WDIO_SETTINGS'; + +// WDIO loads this configuration in the launcher and in every worker. It is constant: the run's settings and the +// configuration module travel as data in a JSON file named by the environment, so nothing is spliced into code that runs. +const generatedConfiguration = `import fs from 'node:fs'; +const settingsPath = process.env.DESKTOP_WDIO_SETTINGS; +if (!settingsPath) throw new Error('DESKTOP_WDIO_SETTINGS must name the desktop WDIO settings file.'); +const { configurationModule, settings } = JSON.parse(fs.readFileSync(settingsPath, 'utf8')); +const { createDesktopWdioConfiguration } = await import(configurationModule); +export const config = await createDesktopWdioConfiguration(settings); +`; + /** A lane selects tests through the Jasmine adapter, so a suite on another framework cannot honor one. */ export function assertDesktopSuiteSupportsInput(suiteName: string, suite: DesktopWdioSuite, input: DesktopInputLane | undefined): void { if (input && suite.framework !== 'jasmine') { @@ -66,14 +78,13 @@ export async function runDesktopWdioSuite( launchTimeoutMs: supervisor.target.launchTimeoutMs ?? 180_000, readiness: suite.readiness ?? supervisor.target.readiness, }; + const settingsPath = supervisor.reporter.artifactPath('wdio.settings.json'); + fs.writeFileSync(settingsPath, JSON.stringify({ configurationModule: new URL('./configuration.js', import.meta.url).href, settings }), { + mode: 0o600, + }); // WDIO initializes its TypeScript loader before evaluating TypeScript configurations and their hooks. const configPath = supervisor.reporter.artifactPath('wdio.generated.mts'); - fs.writeFileSync( - configPath, - `import { createDesktopWdioConfiguration } from ${JSON.stringify(new URL('./configuration.js', import.meta.url).href)};\n` + - `export const config = await createDesktopWdioConfiguration(${JSON.stringify(settings)});\n`, - { mode: 0o600 }, - ); + fs.writeFileSync(configPath, generatedConfiguration, { mode: 0o600 }); const launcher = modulePath('@wdio/cli'); // The selection travels as data in the environment: a worker parses it, and nothing is spliced into code that runs. const selection = serializeDesktopTestSelection({ @@ -87,7 +98,12 @@ export async function runDesktopWdioSuite( command: process.execPath, args: [fileURLToPath(new URL('../../config/launcher.cjs', import.meta.url)), pathToFileURL(launcher).href, configPath], cwd: config.projectRoot, - env: { E2ETEST_PLATFORM: supervisor.target.endpoint, NATIVE_TESTING: 'false', [desktopTestSelectionEnvironment]: selection }, + env: { + E2ETEST_PLATFORM: supervisor.target.endpoint, + NATIVE_TESTING: 'false', + [desktopTestSelectionEnvironment]: selection, + [desktopWdioSettingsEnvironment]: settingsPath, + }, label: `WDIO ${suiteName} (${supervisor.target.endpoint})`, }, suiteDeadline(suite, specs.length), @@ -103,5 +119,6 @@ export async function runDesktopWdioSuite( function suiteDeadline(suite: DesktopWdioSuite, groups: number): number { // A suite deadline bounds a wedged launcher; Jasmine still enforces its independent per-test timeout. + if (suite.suiteTimeoutMs !== undefined) return suite.suiteTimeoutMs; return Math.min(2_147_483_647, Math.max(300_000, (suite.timeoutMs ?? 60_000) * 100 * groups * ((suite.specFileRetries ?? 0) + 1))); } diff --git a/packages/agentic/storybook-desktop/config/server.cjs b/packages/agentic/storybook-desktop/config/server.cjs index a87b87503d..2e1ad31efd 100644 --- a/packages/agentic/storybook-desktop/config/server.cjs +++ b/packages/agentic/storybook-desktop/config/server.cjs @@ -24,11 +24,13 @@ async function startDesktopStorybookServer({ }); let stopping = false; const controller = new AbortController(); - const [{ startNativeDesktopService }, { createDesktopReportSession }, { NativeDesktopHost }] = await Promise.all([ - import('@fluentui-react-native/desktop-test/driver-service'), - import('@fluentui-react-native/desktop-test/reporting'), - import('@fluentui-react-native/desktop-driver'), - ]); + const [{ handleDesktopInterrupts }, { startNativeDesktopService }, { createDesktopReportSession }, { NativeDesktopHost }] = + await Promise.all([ + import('@fluentui-react-native/desktop-test'), + import('@fluentui-react-native/desktop-test/driver-service'), + import('@fluentui-react-native/desktop-test/reporting'), + import('@fluentui-react-native/desktop-driver'), + ]); const report = await createDesktopReportSession({ artifactsRoot: path.join(projectRoot, 'artifacts', 'desktop'), kind: 'lifecycle' }); let server; let controlledChannel; @@ -38,8 +40,6 @@ async function startDesktopStorybookServer({ let closing; const close = (error) => (closing ??= (async () => { - process.off('SIGINT', stop); - process.off('SIGTERM', stop); const failures = error === undefined ? [] : [error]; try { // Compatibility breadcrumb only; correlated process readiness is authoritative. @@ -79,7 +79,7 @@ async function startDesktopStorybookServer({ unregisterDisposal(); if (failures.length) throw new AggregateError(failures, 'Storybook channel disposal is incomplete.'); finishDesktopProcess(); - })()); + })().finally(() => releaseInterrupts?.())); const settle = async (error) => { stopping = true; controller.abort(error ?? new Error('Storybook channel stop requested.')); @@ -102,8 +102,18 @@ async function startDesktopStorybookServer({ }; const unregisterDisposal = registerDesktopProcessDisposal(() => settle()); report.onStop?.(stop); - process.once('SIGINT', stop); - process.once('SIGTERM', stop); + // Handlers stay installed until close completes; nothing can be forced, so a repeated signal only warns before exiting. + const releaseInterrupts = handleDesktopInterrupts({ + stop, + terminate: (signal) => + report.reporter.emit({ + kind: 'diagnostic', + payload: { + level: 'warning', + message: `Received ${signal} while the Storybook channel is closing. Interrupt again to exit without cleanup.`, + }, + }), + }); try { controller.signal.throwIfAborted(); const driverManifest = driverManifestPath ? JSON.parse(fs.readFileSync(driverManifestPath, 'utf8')) : undefined; diff --git a/packages/agentic/storybook-desktop/references/commands.md b/packages/agentic/storybook-desktop/references/commands.md index 8e63384484..e1c9a63fd7 100644 --- a/packages/agentic/storybook-desktop/references/commands.md +++ b/packages/agentic/storybook-desktop/references/commands.md @@ -89,7 +89,11 @@ cannot be combined with an explicit non-`auto` click mode. See The TypeScript API exposes the same operations through `DesktopStorybookCli`. Higher-level supervisors should inject their shared reporter, runner, and -services so runs join one owned lifecycle. +services so runs join one owned lifecycle. Injected services are borrowed: the +CLI stops its work through the scope signal but never closes that scope, which +stays with the supervisor that created it. A `DesktopStorybookCli` that creates +its own scope closes it when the command ends. Stage commands receive the scope +signal, so a stop or interrupt ends a running stage instead of waiting for it. JSON mode emits one compact command result, or one readiness record for the long-running `driver` and `server` commands. Their readiness records are @@ -126,9 +130,11 @@ Structured modes keep normal stderr empty. The Node worker reporter is named `--test-reporter spec|tap|dot`. The former `test --reporter spec|tap|dot` spelling remains as a deprecated one-cycle alias; use the global `--reporter` for CLI output format. -SIGINT and SIGTERM stop owned work; one-shot structured runs end with an +The first SIGINT, SIGTERM, or SIGHUP stops owned work; a second one during +cleanup force-stops owned commands, and a third exits without cleanup. +Interrupted one-shot commands, such as a stage or story test, end with an `incomplete` summary, while long-lived commands keep their single readiness -record. +record. A standalone Storybook channel closes cleanly on the same signals. Use `desktop-driver stories list` for runtime story IDs. Static manifest discovery is not a replacement for querying the running catalog. Generic diff --git a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts index d8eeec0efc..f55799763d 100644 --- a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts +++ b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts @@ -833,6 +833,31 @@ describe('DesktopStorybookCli', () => { // eslint-disable-next-line @microsoft/sdl/no-insecure-url -- React Native Test App connects to loopback Metro const fixedWindowsMetroUrl = 'http://127.0.0.1:8081/status'; + test('leaves a borrowed scope to the integration that tracks the run in it', async () => { + const manifestPath = path.join(storybookRoot, 'storybook-desktop.generated', 'driver-manifest.windows.json'); + const runner = new RecordingRunner(); + const services = new DesktopServiceScope(runner); + const close = jest.spyOn(services, 'close'); + const cli = new DesktopStorybookCli(makeConfig({ windows: { smoke: { metroUrl: fixedWindowsMetroUrl } } }), { + ...nativeDriverTestOptions, + createStoryManifest: createEmptyStoryManifest, + fetch: async () => new Response('{}'), + isPortAvailable: async () => true, + onReady: () => services.requestStop(), + output: { write: () => true }, + runner, + services, + }); + try { + // desktop-test integrations guard their run in the scope they lend; closing it from inside would wait on itself. + await expect(services.guard(cli.driver('windows'))).rejects.toThrow('Owned desktop service stop requested.'); + expect(close).not.toHaveBeenCalled(); + await expect(services.close()).resolves.toBeUndefined(); + } finally { + fs.rmSync(manifestPath, { force: true }); + } + }); + test('serves Metro on the fixed port a Windows Debug host reads', async () => { const manifestPath = path.join(storybookRoot, 'storybook-desktop.generated', 'driver-manifest.windows.json'); const runner = new RecordingRunner(); diff --git a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts index 76546ef8b6..48d478571e 100644 --- a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts +++ b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts @@ -146,6 +146,9 @@ export class DesktopStorybookCli { private readonly reporterMode?: ResolvedDesktopReporterMode; private readonly structuredOutput: Pick; private readonly services: DesktopServiceScope; + // A borrowed scope belongs to its caller, which may be awaiting this command inside the scope's tracked work; closing + // it here would wait on that work. The caller closes it. + private readonly ownsServices: boolean; private readonly onReady?: () => void; constructor(config: DesktopStorybookConfig, options: DesktopStorybookCliOptions = {}) { @@ -167,6 +170,7 @@ export class DesktopStorybookCli { verbose: options.verbose, }); this.services = options.services ?? new DesktopServiceScope(this.runner, this.reporter); + this.ownsServices = options.services === undefined; this.buildNativeDriver = options.buildNativeDriver ?? buildNativeDesktopDriver; this.createStoryManifest = options.createStoryManifest ?? createDesktopStoryManifest; this.fetch = options.fetch ?? globalThis.fetch; @@ -222,7 +226,7 @@ export class DesktopStorybookCli { failures.push(error); } try { - await this.services.close(); + if (this.ownsServices) await this.services.close(); } catch (error) { failures.push(error); } @@ -324,7 +328,7 @@ export class DesktopStorybookCli { failures.push(error); } try { - await this.services.close(); + if (this.ownsServices) await this.services.close(); } catch (error) { failures.push(error); } @@ -689,7 +693,7 @@ export class DesktopStorybookCli { failures.push(error); } try { - await this.services.close(); + if (this.ownsServices) await this.services.close(); } catch (error) { failures.push(error); } @@ -1024,9 +1028,12 @@ export class DesktopStorybookCli { cleanup = false, ): Promise { for (const command of Array.isArray(plan) ? plan : [plan]) { - const work = this.runner.run(this.prepareCommand(command, platform, instance)); - if (cleanup) await work; - else await this.services.guard(work); + // Cleanup runs after the scope has stopped, so only regular steps are cancelled with it. + if (cleanup) await this.runner.run(this.prepareCommand(command, platform, instance)); + else + await this.services.guard( + this.runner.run(this.prepareCommand(command, platform, instance), { signal: this.services.controller.signal }), + ); } } diff --git a/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts b/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts index 88ae7e784a..bb8d43e5cc 100644 --- a/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts +++ b/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts @@ -1,6 +1,12 @@ -import { Command, Option } from 'commander'; +import { Command, InvalidArgumentError, Option } from 'commander'; +import { randomUUID } from 'node:crypto'; import path from 'node:path'; -import { claimDesktopWorkload, DesktopServiceScope, DesktopStopRequested } from '@fluentui-react-native/desktop-test'; +import { + claimDesktopWorkload, + DesktopServiceScope, + DesktopStopRequested, + handleDesktopInterrupts, +} from '@fluentui-react-native/desktop-test'; import { desktopEventLevels, desktopReporterModes, @@ -136,7 +142,7 @@ function addTestCommand(program: Command, getApi: GetApi): void { addPlatformOptions(command); command.action(async (flags: PlatformFlags & DesktopStorybookTestOptions) => { if (flags.input !== undefined && flags.clickMode !== undefined) { - throw new TypeError('--input cannot be combined with --click-mode.'); + throw new InvalidArgumentError('--input cannot be combined with --click-mode.'); } const api = await getApi(flags); await api.test(resolvePlatform(flags, api), { @@ -189,7 +195,7 @@ function addDriverCommand(program: Command, getApi: GetApi): void { addPlatformOptions(command); command.action(async (flags: DriverFlags) => { if (!flags.agent && (flags.descriptor !== undefined || flags.root !== undefined)) { - throw new TypeError('--descriptor and --root require --agent.'); + throw new InvalidArgumentError('--descriptor and --root require --agent.'); } const api = await getApi(flags); await api.driver(resolvePlatform(flags, api), { @@ -237,9 +243,11 @@ export async function runDesktopStorybookCli(argv: readonly string[] = process.a let session: DesktopReportSession | undefined; let runner: NodeDesktopCommandRunner | undefined; let releaseClaim: (() => void) | undefined; + let releaseInterrupts: (() => void) | undefined; + let actionName = ''; + let longLived = false; let failure: unknown; let parseDiagnostics = ''; - const interrupt = () => options.services?.requestStop(); const normalized = normalizeLegacyTestReporter(argv); let reporterResolutionError: unknown; try { @@ -294,6 +302,8 @@ export async function runDesktopStorybookCli(argv: readonly string[] = process.a : undefined, }) : undefined; + actionName = action.name(); + longLived = ['driver', 'server'].includes(actionName); session = await createDesktopReportSession({ ...options.config.desktop?.reporting, artifactsRoot: options.config.desktop?.reporting.artifactsRoot ?? path.join(options.config.projectRoot, 'artifacts', 'desktop'), @@ -302,7 +312,7 @@ export async function runDesktopStorybookCli(argv: readonly string[] = process.a reporterMode, eventLevel, // A long-lived command's readiness record is its only output; before readiness, a failure reports its summary. - suppressFinalOutput: ['driver', 'server'].includes(action.name()) ? () => ready : false, + suppressFinalOutput: longLived ? () => ready : false, verbose: program.opts<{ verbose?: boolean }>().verbose, onFatal: (error) => options.services?.fail(error), }); @@ -322,31 +332,52 @@ export async function runDesktopStorybookCli(argv: readonly string[] = process.a options.runner = runner; options.services = new DesktopServiceScope(runner, session.reporter); session.onStop?.(() => options.services!.requestStop()); - process.once('SIGINT', interrupt); - process.once('SIGTERM', interrupt); + const reporter = session.reporter; + releaseInterrupts = handleDesktopInterrupts({ + stop: () => options.services?.requestStop(), + terminate: (signal) => { + reporter.emit({ + kind: 'diagnostic', + payload: { + level: 'warning', + message: `Received ${signal} during shutdown; force-stopping owned commands. Interrupt again to exit without cleanup.`, + }, + }); + runner?.terminate(); + }, + }); }); try { await program.parseAsync([...normalized.argv]); } catch (error) { if (error && typeof error === 'object' && 'exitCode' in error && error.exitCode === 0) return; if (!(error instanceof DesktopStopRequested)) failure = reporterResolutionError ?? error; + else if (session && !longLived) + // An interrupted one-shot command did not finish: leave its scope open so the run reports incomplete, not passed. + session.reporter.emit( + { kind: 'scope-start', payload: { label: `${actionName} (interrupted)`, category: 'phase' } }, + `interrupted-${randomUUID()}`, + ); } finally { - process.off('SIGINT', interrupt); - process.off('SIGTERM', interrupt); - try { - await options.services?.close(); - await runner?.close(); - } catch (error) { - options.reporter?.fail(error, 'cleanup'); - failure = failure === undefined ? error : new AggregateError([failure, error], 'Storybook execution and cleanup failed.'); - } + // Interrupt handlers stay installed until cleanup completes, so a repeated signal escalates instead of killing the CLI. try { - releaseClaim?.(); - } catch (error) { - options.reporter?.fail(error, 'cleanup'); - failure ??= error; + try { + await options.services?.close(); + await runner?.close(); + } catch (error) { + options.reporter?.fail(error, 'cleanup'); + failure = failure === undefined ? error : new AggregateError([failure, error], 'Storybook execution and cleanup failed.'); + } + try { + releaseClaim?.(); + } catch (error) { + options.reporter?.fail(error, 'cleanup'); + failure ??= error; + } + if (session) process.exitCode = await session.close(failure); + } finally { + releaseInterrupts?.(); } - if (session) process.exitCode = await session.close(failure); } if (failure !== undefined) { if (options.reporterMode === 'json' || options.reporterMode === 'ndjson') { @@ -417,7 +448,7 @@ function addSmokeCommand(program: Command, getApi: GetApi): void { addPlatformOptions(command); command.action(async (flags: SmokeFlags) => { if (flags.input !== undefined && flags.mode !== 'stories-and-tests') { - throw new TypeError('--input requires --mode stories-and-tests.'); + throw new InvalidArgumentError('--input requires --mode stories-and-tests.'); } const api = await getApi(flags); await api.smoke(resolvePlatform(flags, api), { mode: flags.mode, input: flags.input }); @@ -475,7 +506,7 @@ function resolvePlatform(flags: PlatformFlags, api: DesktopStorybookCli): Platfo function parsePort(value: string): number { const port = Number(value); if (!Number.isInteger(port) || port < 1 || port > 65535) { - throw new TypeError(`Server port must be an integer between 1 and 65535. Received "${value}".`); + throw new InvalidArgumentError(`Server port must be an integer between 1 and 65535. Received "${value}".`); } return port; } diff --git a/packages/agentic/storybook-desktop/src/cli/diagnostics.test.ts b/packages/agentic/storybook-desktop/src/cli/diagnostics.test.ts index 5f35f69fc5..8349ebf530 100644 --- a/packages/agentic/storybook-desktop/src/cli/diagnostics.test.ts +++ b/packages/agentic/storybook-desktop/src/cli/diagnostics.test.ts @@ -87,9 +87,14 @@ describe('pipeline failure diagnostics', () => { }); test.each([ - ['usage', ['test', '--macos', '--reporter', 'json', '--unknown']], - ['config', ['--reporter', 'json', '--config', 'missing-storybook.config.mjs', 'manifest', '--macos']], - ])('emits one structured %s error without stderr', async (_kind, args) => { + { label: 'unknown test option', kind: 'usage', args: ['test', '--macos', '--reporter', 'json', '--unknown'] }, + { label: 'subcommand option value', kind: 'usage', args: ['--reporter', 'ndjson', 'server', '--port', 'abc'] }, + { + label: 'config file', + kind: 'config', + args: ['--reporter', 'json', '--config', 'missing-storybook.config.mjs', 'manifest', '--macos'], + }, + ])('emits one structured $kind error without stderr for $label', async ({ kind, args }) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'storybook-cli-structured-error-')); try { const response = await new Promise<{ code: number | null; stdout: string; stderr: string }>((resolve, reject) => { @@ -111,7 +116,7 @@ describe('pipeline failure diagnostics', () => { expect(JSON.parse(response.stdout)).toMatchObject({ schemaVersion: 1, type: 'error', - error: { code: _kind }, + error: { code: kind }, }); } finally { fs.rmSync(root, { recursive: true, force: true }); diff --git a/packages/agentic/storybook-desktop/src/cli/interrupts.test.ts b/packages/agentic/storybook-desktop/src/cli/interrupts.test.ts new file mode 100644 index 0000000000..bfd75342ab --- /dev/null +++ b/packages/agentic/storybook-desktop/src/cli/interrupts.test.ts @@ -0,0 +1,96 @@ +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +jest.setTimeout(30_000); + +// Constant config: the stage command ignores SIGTERM, so only the CLI's second-interrupt escalation can stop it before the +// runner's 10-second graceful-stop deadline. Paths are derived at runtime, never spliced into the source (R-SCAN-002). +const config = ` +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +const projectRoot = path.dirname(fileURLToPath(import.meta.url)); +const fixture = "process.on('SIGTERM', () => {}); require('node:fs').writeFileSync(process.argv[1], String(process.pid)); setInterval(() => {}, 1000);"; +export default { + projectRoot, + getStorybookConfig: () => ({}), + getPlatformOptions: () => ({}), + getStoryPackages: () => [], + getCommandPlan: () => ({ command: process.execPath, args: ['-e', fixture, path.join(projectRoot, 'stage.pid')] }), + getSmokeOptions: () => false, + getNativeDriverOptions: () => undefined, + resolvePackage: () => projectRoot, +}; +`; + +const posixTest = process.platform === 'win32' ? test.skip : test; + +describe('storybook CLI interrupts', () => { + posixTest.each([ + ['SIGINT', 'SIGINT'], + ['SIGHUP', 'SIGTERM'], + ] as const)( + 'keeps handling interrupts during cleanup: %s then %s force-stops the stage and reports the run incomplete', + async (first, second) => { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'storybook-cli-interrupts-'))); + const configPath = path.join(root, 'storybook.config.mjs'); + const markerPath = path.join(root, 'stage.pid'); + fs.writeFileSync(configPath, config); + let stagePid: number | undefined; + const child = spawn( + process.execPath, + [path.resolve(__dirname, '../../config/cli.cjs'), '--config', configPath, 'bundle', '--macos'], + { + cwd: root, + env: { ...process.env, DESKTOP_REPORTER: 'plain' }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + const output: Buffer[] = []; + child.stdout.on('data', (chunk: Buffer) => output.push(chunk)); + child.stderr.on('data', (chunk: Buffer) => output.push(chunk)); + const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => + child.once('exit', (code, signal) => resolve({ code, signal })), + ); + try { + await waitUntil(() => fs.existsSync(markerPath) && fs.readFileSync(markerPath, 'utf8').length > 0); + stagePid = Number(fs.readFileSync(markerPath, 'utf8')); + child.kill(first); + // The stage ignores the graceful stop, so cleanup is still waiting on it when the second interrupt arrives. + await new Promise((resolve) => setTimeout(resolve, 500)); + expect(isAlive(stagePid)).toBe(true); + const escalated = Date.now(); + child.kill(second); + const result = await exited; + const text = Buffer.concat(output).toString('utf8'); + expect(result).toEqual({ code: 1, signal: null }); + expect(text).toContain('force-stopping owned commands'); + expect(text).toContain('[desktop] INCOMPLETE: lifecycle'); + expect(Date.now() - escalated).toBeLessThan(8_000); + await waitUntil(() => !isAlive(stagePid!)); + } finally { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + if (stagePid && isAlive(stagePid)) process.kill(stagePid, 'SIGKILL'); + fs.rmSync(root, { force: true, recursive: true, maxRetries: 20, retryDelay: 100 }); + } + }, + ); +}); + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'EPERM'; + } +} + +async function waitUntil(condition: () => boolean, timeoutMs = 15_000): Promise { + const deadline = Date.now() + timeoutMs; + while (!condition()) { + if (Date.now() > deadline) throw new Error('Timed out waiting for the condition.'); + await new Promise((resolve) => setTimeout(resolve, 50)); + } +} diff --git a/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts b/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts index 142615b7d1..d140f24ca9 100644 --- a/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts +++ b/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts @@ -1,3 +1,4 @@ +import { spawn } from 'node:child_process'; import fs from 'node:fs'; import { createServer, Socket } from 'node:net'; import os from 'node:os'; @@ -216,6 +217,66 @@ describe('desktop Storybook server integration', () => { } }); + // An owned channel is disposed by its parent; a standalone one must still clean up on a terminal hangup. + (process.platform === 'win32' ? test.skip : test)('closes a standalone channel cleanly on SIGHUP', async () => { + const [storybookPort, driverPort] = await Promise.all([getAvailablePort(), getAvailablePort()]); + const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-storybook-driver-')); + const manifestPath = path.join(temporaryDirectory, 'driver-manifest.json'); + const projectRoot = path.resolve(__dirname, '../../../../../apps/storybook'); + fs.writeFileSync(manifestPath, JSON.stringify(createIntegrationManifest(temporaryDirectory, storybookPort, driverPort))); + const output: string[] = []; + const sink = { + write: (text: string) => { + output.push(text); + return true; + }, + }; + const reporter = new DesktopRunReporter({ + artifactsRoot: temporaryDirectory, + kind: 'lifecycle', + reporterMode: 'plain', + output: sink, + errorOutput: sink, + summary: false, + }); + const transport = await startReportTransport(reporter); + const child = spawn(process.execPath, [path.resolve(__dirname, '../../config/server-runner.cjs')], { + cwd: projectRoot, + env: { + ...process.env, + ...transport.environment, + STORYBOOK_CONFIG_PATH: path.join(projectRoot, 'src'), + STORYBOOK_DRIVER_MANIFEST: manifestPath, + STORYBOOK_NATIVE_DRIVER_FAKE: '1', + STORYBOOK_PROJECT_ROOT: projectRoot, + STORYBOOK_WS_PORT: String(storybookPort), + }, + stdio: 'ignore', + }); + const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => + child.once('exit', (code, signal) => resolve({ code, signal })), + ); + try { + await deadline( + (async () => { + while (!output.join('').includes('READY Storybook')) await new Promise((resolve) => setTimeout(resolve, 50)); + })(), + 20_000, + 'Waiting for the standalone channel', + ); + child.kill('SIGHUP'); + expect(await exited).toEqual({ code: 0, signal: null }); + } finally { + try { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + await transport.close(); + expect((await reporter.finish()).status).toBe('passed'); + } finally { + fs.rmSync(temporaryDirectory, { force: true, recursive: true }); + } + } + }); + test('refuses an agent connection from the fake driver', async () => { const [storybookPort, driverPort] = await Promise.all([getAvailablePort(), getAvailablePort()]); const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-storybook-driver-'));