From 53f276c81570e50ea88609febebfaaecb908b1a2 Mon Sep 17 00:00:00 2001 From: Jason Morse Date: Sun, 8 Nov 2026 16:09:26 -0800 Subject: [PATCH 1/2] Improve desktop-test and storybook-desktop logging output Add runDesktopOperation for grouped, collapsed-on-success operation output, add smoke-test run summaries, and replace stop-commands fencing with per-line workflow-command neutralization. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9305975d-a017-4007-beb2-68a4684cbbc4 --- .changeset/desktop-test-operation-logging.md | 5 + .changeset/storybook-desktop-smoke-summary.md | 5 + packages/agentic/desktop-test/jest.config.cjs | 2 +- packages/agentic/desktop-test/package.json | 1 - .../desktop-test/references/commands.md | 5 + .../desktop-test/references/reporting.md | 56 +++- packages/agentic/desktop-test/src/cli.ts | 2 +- packages/agentic/desktop-test/src/commands.ts | 46 ++- .../desktop-test/src/contracts.contract.cjs | 59 +++- .../src/reporting/collector.test.ts | 275 ++++++++++++++++++ .../desktop-test/src/reporting/collector.ts | 215 ++++++++------ .../desktop-test/src/reporting/index.ts | 2 + .../src/reporting/operation.test.ts | 224 ++++++++++++++ .../desktop-test/src/reporting/operation.ts | 65 +++++ .../src/reporting/presentation.test.ts | 215 ++++++++++++++ .../src/reporting/presentation.ts | 164 +++++++++++ .../src/reporting/protocol.test.ts | 6 + .../desktop-test/src/reporting/protocol.ts | 16 +- .../src/reporting/structured.test.ts | 8 + .../desktop-test/src/reporting/structured.ts | 1 + packages/agentic/desktop-test/src/services.ts | 7 +- .../config/storybook-control.cjs | 126 ++++---- .../agentic/storybook-desktop/jest.config.cjs | 2 +- .../storybook-desktop/references/commands.md | 10 + .../references/platforms/macos.md | 5 +- .../references/troubleshooting.md | 10 + .../src/cli/DesktopStorybookCli.test.ts | 144 ++++++++- .../src/cli/DesktopStorybookCli.ts | 91 ++++-- .../src/cli/commandRunner.test.ts | 46 ++- .../src/cli/createDesktopStorybookCommand.ts | 2 +- .../src/cli/smokeTests.test.ts | 80 ++++- .../storybook-desktop/src/cli/smokeTests.ts | 35 ++- .../src/driver/serverIntegration.test.ts | 41 ++- .../src/testing/laneVerdict.test.ts | 16 +- .../src/testing/laneVerdict.ts | 12 +- yarn.lock | 1 - 36 files changed, 1756 insertions(+), 244 deletions(-) create mode 100644 .changeset/desktop-test-operation-logging.md create mode 100644 .changeset/storybook-desktop-smoke-summary.md create mode 100644 packages/agentic/desktop-test/src/reporting/operation.test.ts create mode 100644 packages/agentic/desktop-test/src/reporting/operation.ts create mode 100644 packages/agentic/desktop-test/src/reporting/presentation.test.ts create mode 100644 packages/agentic/desktop-test/src/reporting/presentation.ts diff --git a/.changeset/desktop-test-operation-logging.md b/.changeset/desktop-test-operation-logging.md new file mode 100644 index 0000000000..e9122fbc12 --- /dev/null +++ b/.changeset/desktop-test-operation-logging.md @@ -0,0 +1,5 @@ +--- +'@fluentui-react-native/desktop-test': patch +--- + +Report each command and step as a one-line outcome header with its duration and summary. Command output stays in its log and shows only when the step fails, in a verbose run, or inside a collapsed GitHub Actions group, and `runDesktopOperation` exposes the same headers to other producers. Neutralize workflow commands per line instead of wrapping every message in `::stop-commands::` fences, drop the per-message command identifiers, and remove the unused `@rnx-kit/reporter` dependency. diff --git a/.changeset/storybook-desktop-smoke-summary.md b/.changeset/storybook-desktop-smoke-summary.md new file mode 100644 index 0000000000..72c5cbaf53 --- /dev/null +++ b/.changeset/storybook-desktop-smoke-summary.md @@ -0,0 +1,5 @@ +--- +'@fluentui-react-native/storybook-desktop': patch +--- + +Run story rendering and story tests as reported operations, so a smoke run shows one header with counts such as `209 stories rendered` and `101 of 101 story tests executed: 101 passed, 0 failed, 0 skipped`, and says `no story tests executed` when none ran. `--verbose` streams command output live; other runs keep it in the logs and show the end of a failing step's log. diff --git a/packages/agentic/desktop-test/jest.config.cjs b/packages/agentic/desktop-test/jest.config.cjs index 8f56587ad6..b28d09e48f 100644 --- a/packages/agentic/desktop-test/jest.config.cjs +++ b/packages/agentic/desktop-test/jest.config.cjs @@ -2,7 +2,7 @@ const config = require('@fluentui-react-native/scripts/jest-config'); module.exports = { ...config, - transformIgnorePatterns: ['node_modules[/\\\\](?!.*(?:\\.mjs$|@rnx-kit(?:[/\\\\]reporter|-reporter-npm-)))'], + transformIgnorePatterns: ['node_modules[/\\\\](?!.*\\.mjs$)'], moduleNameMapper: { ...config.moduleNameMapper, '^(\\.{1,2}/.*)\\.js$': '$1', diff --git a/packages/agentic/desktop-test/package.json b/packages/agentic/desktop-test/package.json index 6fa9a882f5..31512c19fd 100644 --- a/packages/agentic/desktop-test/package.json +++ b/packages/agentic/desktop-test/package.json @@ -92,7 +92,6 @@ }, "dependencies": { "@fluentui-react-native/desktop-driver": "workspace:*", - "@rnx-kit/reporter": "catalog:", "@wdio/reporter": "catalog:", "commander": "^14.0.2", "cross-spawn": "7.0.6", diff --git a/packages/agentic/desktop-test/references/commands.md b/packages/agentic/desktop-test/references/commands.md index abaa38af42..0dcec90136 100644 --- a/packages/agentic/desktop-test/references/commands.md +++ b/packages/agentic/desktop-test/references/commands.md @@ -25,6 +25,11 @@ and `--json` (an alias for `--reporter json`). Mode precedence is CLI flag, `DESKTOP_REPORTER`, config, then `auto`. `auto` selects GitHub Actions rendering when `GITHUB_ACTIONS=true`, otherwise plain output. +Plain and GitHub Actions output reports each command as one header with its +outcome and duration, and shows the end of its log only when it fails. Pass +`--verbose` to stream every command's output live, with start markers and log +paths. See [console output](reporting.md#console-output). + One-shot JSON commands emit a bounded summary; `--json` no longer prints the complete `run.json`. NDJSON starts with run metadata, emits milestone records by default, then ends with the same summary. Use `--events progress|all` for diff --git a/packages/agentic/desktop-test/references/reporting.md b/packages/agentic/desktop-test/references/reporting.md index 1c0c7f64f1..47661bc059 100644 --- a/packages/agentic/desktop-test/references/reporting.md +++ b/packages/agentic/desktop-test/references/reporting.md @@ -24,16 +24,64 @@ are flags, not credentials, and redacting them would rewrite machine-readable output. Register a shorter credential explicitly with the reporter's `addSecret`. -Stderr and test progress are live without verbose mode. Failed commands replay -the collected output. GitHub Actions grouping and annotations are rendered by -the supervisor; child output cannot issue workflow commands. Upload the full -run directory in an always-run artifact step. +Command output goes to these logs, not to the console; see +[console output](#console-output) for what the console shows and when. Upload +the full run directory in an always-run artifact step. Statuses preserve failures, retries, flaky tests, skips, unsupported capabilities, and unfinished work. A zero worker exit cannot override failed assertions or incomplete work. Optional suites can be skipped without being labelled passed. +## Console output + +A command, or a step such as a story test run, is an operation. When it ends the +reporter prints one header with its outcome and duration, and a summary when the +operation has one: + +```text +✔ pod install (42s) +✔ Render stories (1m 08s): 209 stories rendered +✔ Story tests (3m 12s): 101 of 101 story tests executed: 101 passed, 0 failed, 0 skipped +✖ rnx-cli build (4m 03s): exit code 65 +``` + +The operation's output stays in its log, and the console shows it only when it +helps: + +- **Default.** A passing operation prints its header on stdout. A failing one + prints its header on stderr, the last 200 lines of its log indented beneath + it, and `Full log: `. +- **GitHub Actions.** A passing operation that has a log prints a collapsed + `::group::` titled with the header, holding the last 1000 lines of the log and + `Full log: `. A failing operation prints as it does by default, so an + error is never folded away, and each failure also gets an `::error` + annotation. +- **`--verbose`.** The operation's output streams live as `[label] stdout: text` + and `[label] stderr: text`, after a `▶ label` start line and an `artifact:` + line naming its log. The header follows, and a failing operation adds + `Full log: `. + +Without `--verbose` a passing, skipped, or unsupported test appears only in the +final counts, so a run that executes nothing shows as `0 executed` instead of +disappearing into a long list. Failed tests, retries, and diagnostics always +print, and the root supervisor's stderr is always live. Every run finishes with a +`[desktop] PASSED: …` or `[desktop] FAILED: …` line carrying the counts. + +Output is untrusted text. The reporter neutralizes each line it prints that it +did not build as a workflow command, so a command cannot start, stop, or forge +one: `::` at the start of a line (after any white space) becomes `: :`, and `##[` +becomes `# #[`. Only text that would have been a command changes, and the logs +keep the original. The console has no `::stop-commands::` fence, per-message +token lines, or command identifiers; agents that need complete output read the +logs and `run.json`. + +Delegated producers and the Storybook CLI report operations the same way through +`runDesktopOperation` from `@fluentui-react-native/desktop-test/reporting`. Its +`summarize` sets the header's summary. Its `log` records a line of detail that is +live only with `--verbose`; it is not kept in a log, so it is not replayed when +the operation fails. + ## CLI reporters Select `--reporter auto|plain|github-actions|json|ndjson`; `--json` aliases diff --git a/packages/agentic/desktop-test/src/cli.ts b/packages/agentic/desktop-test/src/cli.ts index 659800eb62..cc2a039294 100644 --- a/packages/agentic/desktop-test/src/cli.ts +++ b/packages/agentic/desktop-test/src/cli.ts @@ -35,7 +35,7 @@ export async function runDesktopTestCli(argv = process.argv): Promise { .option('--json', 'alias for --reporter json') .addOption(new Option('--events ', 'NDJSON event detail').choices([...desktopEventLevels])) .option('--paper', 'select macOS Paper (run prep after switching renderers)') - .option('--verbose', 'show informational service output'); + .option('--verbose', 'stream command output live, with start markers and log paths, instead of only the outcome of each step'); let parseDiagnostics = ''; program.exitOverride().configureOutput({ writeErr: (message) => { diff --git a/packages/agentic/desktop-test/src/commands.ts b/packages/agentic/desktop-test/src/commands.ts index d8341be369..6407a365a4 100644 --- a/packages/agentic/desktop-test/src/commands.ts +++ b/packages/agentic/desktop-test/src/commands.ts @@ -66,9 +66,7 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { try { const code = await running.completed; if (code !== 0) - throw ( - running.failure ?? new Error(`Command "${command.label ?? command.command}" exited with code ${code}. Log: ${running.logPath}`) - ); + throw running.failure ?? new Error(`Command "${commandLabel(command)}" exited with code ${code}. Log: ${running.logPath}`); } catch (error) { failure = error; } @@ -96,6 +94,7 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { reporter.addSecret(value); } const id = `command-${randomUUID()}`; + const label = commandLabel(command); const emit = reporter.source(id); const relativeLog = `commands/${id}.log`; const logPath = reporter.artifactPath(relativeLog); @@ -104,12 +103,8 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { flag: 'wx', mode: 0o600, }); - emit( - { kind: 'scope-start', payload: { label: command.label ?? command.command, category: command.category ?? 'service' } }, - id, - command.context, - ); - emit({ kind: 'artifact', payload: { path: relativeLog, description: command.label ?? command.command } }, id, command.context); + emit({ kind: 'scope-start', payload: { label, category: command.category ?? 'service' } }, id, command.context); + emit({ kind: 'artifact', payload: { path: relativeLog, description: label } }, id, command.context); let failed: unknown; const processControl = createDesktopProcessControl((error) => { failed ??= error; @@ -132,6 +127,7 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { let stopped = false; let closed = false; let exitFailure: Error | undefined; + let exitSummary: string | undefined; let stopPromise: Promise | undefined; let closeDeadline: ReturnType | undefined; let logBytes = fs.statSync(logPath).size; @@ -190,9 +186,7 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { child.once('exit', () => { processControl.stop(); closeDeadline = setTimeout(() => { - failed ??= new Error( - `Child exited but inherited output streams did not close within the drain deadline: ${command.label ?? command.command}`, - ); + failed ??= new Error(`Child exited but inherited output streams did not close within the drain deadline: ${label}`); child.stdout.destroy(); child.stderr.destroy(); }, this.options.shutdownTimeoutMs ?? 10_000); @@ -200,8 +194,10 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { child.once('close', (code, signal) => { clearTimeout(closeDeadline); closed = true; - if (code !== 0 && !stopped) - exitFailure = new Error(`Command "${command.label ?? command.command}" exited with code ${code ?? signal}. Log: ${logPath}`); + if (code !== 0 && !stopped) { + exitSummary = code === null ? `terminated by ${signal}` : `exit code ${code}`; + exitFailure = new Error(`Command "${label}" exited with code ${code ?? signal}. Log: ${logPath}`); + } if (failed !== undefined) reject(failed); else resolve(code ?? (signal ? 1 : 0)); }); @@ -264,7 +260,12 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { } const status = options.failed || failed !== undefined || cleanupError !== undefined || (!wasRunning && code !== 0) ? 'failed' : 'passed'; - emit({ kind: 'scope-end', payload: { status, logPath: relativeLog } }, id, command.context); + const reason = status === 'failed' ? (describeFailure(failed ?? cleanupError) ?? exitSummary) : undefined; + emit( + { kind: 'scope-end', payload: { status, logPath: relativeLog, ...(reason ? { summary: reason } : {}) } }, + id, + command.context, + ); emit({ kind: 'producer-finished', payload: { status } }, id, command.context); this.active.delete(running); await processControl.close(); @@ -283,6 +284,21 @@ export class NodeDesktopCommandRunner implements DesktopCommandRunner { } } +// A first argument such as `install` in `pod install` or `lage` in `yarn lage` names the work. Flags, paths and other values do not. +const subcommandPattern = /^[a-z][a-z0-9-]{0,31}$/i; + +function commandLabel(command: PreparedDesktopCommand): string { + if (command.label) return command.label; + const name = path.basename(command.command); + const [first] = command.args; + return first !== undefined && subcommandPattern.test(first) ? `${name} ${first}` : name; +} + +function describeFailure(error: unknown): string | undefined { + if (error === undefined) return undefined; + return (error instanceof Error ? error.message : String(error)).split('\n', 1)[0]; +} + function needsWindowsCommandShim(command: string): boolean { return ( process.platform === 'win32' && diff --git a/packages/agentic/desktop-test/src/contracts.contract.cjs b/packages/agentic/desktop-test/src/contracts.contract.cjs index e00218fbf8..8296d38e4b 100644 --- a/packages/agentic/desktop-test/src/contracts.contract.cjs +++ b/packages/agentic/desktop-test/src/contracts.contract.cjs @@ -380,7 +380,17 @@ async function main() { const journal = fs.readFileSync(path.join(run.root, 'events.jsonl'), 'utf8'); assert(!journal.includes('secret-value')); assert(journal.includes('root cause')); - assert(err.text().includes('::stop-commands::')); + assert( + !err.text().includes('::stop-commands::') && !out.text().includes('::stop-commands::'), + 'text is defused in place, not fenced', + ); + assert(err.text().includes(': :error::untrusted [REDACTED]'), 'root stderr is shown with its workflow command defused'); + assert( + !err + .text() + .split('\n') + .some((line) => line.trimStart().startsWith('::error::untrusted')), + ); assert.equal((err.text().match(/::error title=/g) || []).length, 1); const junit = fs.readFileSync(path.join(run.root, 'junit.xml'), 'utf8'); assert(junit.includes('attempt')); @@ -523,16 +533,37 @@ async function main() { error: { code: 'unknown-target', message: 'Unknown target "missing".' }, }); } else if (mode === 'commands') { - const run = reporter({ kind: 'lifecycle', console: 'github-actions' }); + const liveOut = capture(); + const liveErr = capture(); + const live = reporter({ kind: 'lifecycle', verbose: true, output: liveOut, errorOutput: liveErr }); + const liveRunner = new NodeDesktopCommandRunner({ reporter: live }); + const liveCommand = liveRunner.start( + command( + "process.stdout.write('ready\\n'); setTimeout(()=>process.stderr.write('live diagnostic\\n'),30); setInterval(()=>{},1000);", + ), + ); + try { + await waitFor(() => liveErr.text().includes('live diagnostic')); + assert(!live.resultSnapshot, 'diagnostic must arrive before shutdown'); + } finally { + await liveCommand.stop(); + } + assert((await live.finish()).status === 'passed'); + assert(liveOut.text().includes('[fixture] stdout: ready'), 'verbose child stdout is live and stays on stdout'); + assert(liveErr.text().includes('[fixture] stderr: live diagnostic')); + assert(!(liveOut.text() + liveErr.text()).includes('[command-'), 'source identifiers do not leak into the display'); + + const quietOut = capture(); + const quietErr = capture(); + const run = reporter({ kind: 'lifecycle', console: 'github-actions', output: quietOut, errorOutput: quietErr }); const runner = new NodeDesktopCommandRunner({ reporter: run }); const running = runner.start( command( - "process.stdout.write('ready\\n'); setTimeout(()=>process.stderr.write('live diagnostic\\n'),30); setInterval(()=>{},1000);", + "process.stdout.write('ready\\n'); setTimeout(()=>process.stderr.write('quiet diagnostic\\n'),30); setInterval(()=>{},1000);", ), ); try { - await waitFor(() => err.text().includes('live diagnostic')); - assert(!run.resultSnapshot, 'diagnostic must arrive before shutdown'); + await waitFor(() => fs.readFileSync(running.logPath, 'utf8').includes('[stderr] quiet diagnostic')); } finally { await running.stop(); } @@ -554,14 +585,16 @@ async function main() { .map((file) => fs.readFileSync(path.join(run.root, 'commands', file), 'utf8')) .join(''); assert(logs.includes('\u20ac')); - assert(logs.includes('[stderr] live diagnostic')); - assert.equal((out.text().match(/::group::/g) || []).length, 2); - assert.equal((out.text().match(/::endgroup::/g) || []).length, 2); - const groupOne = out - .text() - .slice(out.text().indexOf('::group::one'), out.text().indexOf('::endgroup::', out.text().indexOf('::group::one'))); - assert(groupOne.includes('::stop-commands::')); + assert(logs.includes('[stderr] quiet diagnostic')); + assert(!quietErr.text().includes('quiet diagnostic'), 'child output is not live without verbose'); + const quiet = quietOut.text(); + assert.equal((quiet.match(/::group::/g) || []).length, 3, 'each passed operation is one collapsed group'); + assert.equal((quiet.match(/::endgroup::/g) || []).length, 3); + assert(!quiet.includes('::stop-commands::')); + const groupOne = quiet.slice(quiet.indexOf('::group::✔ one '), quiet.indexOf('::endgroup::', quiet.indexOf('::group::✔ one '))); + assert(groupOne.includes('[stdout] ::warning::one')); assert(!groupOne.includes('::warning::two')); + assert(!quiet.split('\n').some((line) => line.startsWith('::warning::')), 'child output cannot start a workflow command'); } else if (mode === 'command-secrets') { const run = reporter({ kind: 'lifecycle' }); const runner = new NodeDesktopCommandRunner({ reporter: run }); @@ -756,7 +789,7 @@ async function main() { ); } else if (mode === 'owned-service') { const { startReportTransport } = await import(lib('reporting/transport.js')); - const run = reporter({ kind: 'lifecycle' }); + const run = reporter({ kind: 'lifecycle', verbose: true }); const transport = await startReportTransport(run); const runner = new NodeDesktopCommandRunner({ reporter: run, environment: transport.environment, shutdownTimeoutMs: 10_000 }); const descriptor = path.join(root, 'service.json'); diff --git a/packages/agentic/desktop-test/src/reporting/collector.test.ts b/packages/agentic/desktop-test/src/reporting/collector.test.ts index 93a7940295..95236ca1aa 100644 --- a/packages/agentic/desktop-test/src/reporting/collector.test.ts +++ b/packages/agentic/desktop-test/src/reporting/collector.test.ts @@ -1,6 +1,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { randomUUID } from 'node:crypto'; import { spawn } from 'node:child_process'; import { once } from 'node:events'; @@ -264,3 +265,277 @@ describe('desktop run reporter final output', () => { }); }); }); + +describe('desktop run reporter operation output', () => { + let artifactsRoot: string; + let out: string; + let err: string; + + beforeEach(() => { + artifactsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-operations-')); + out = ''; + err = ''; + }); + + afterEach(() => { + fs.rmSync(artifactsRoot, { force: true, recursive: true }); + }); + + function start({ + mode = 'plain', + verbose = false, + kind = 'lifecycle', + }: { mode?: 'plain' | 'github-actions'; verbose?: boolean; kind?: 'tests' | 'lifecycle' } = {}): DesktopRunReporter { + return new DesktopRunReporter({ + artifactsRoot, + kind, + output: { write: (chunk: string) => ((out += chunk), true) }, + errorOutput: { write: (chunk: string) => ((err += chunk), true) }, + reporterMode: mode, + summary: false, + verbose, + }); + } + + // A command as the supervisor reports it: a scope named for the command, with its output kept in a log. Returns the log path. + function command( + reporter: DesktopRunReporter, + label: string, + log: string, + { status = 'passed', summary }: { status?: 'passed' | 'failed'; summary?: string } = {}, + ): string { + const id = `command-${randomUUID()}`; + const emit = reporter.source(id); + const relative = `commands/${id}.log`; + fs.mkdirSync(path.join(reporter.root, 'commands'), { recursive: true }); + fs.writeFileSync(path.join(reporter.root, relative), log); + emit({ kind: 'scope-start', payload: { label, category: 'phase' } }, id); + emit({ kind: 'artifact', payload: { path: relative, description: `${label} log` } }, id); + emit({ kind: 'output', payload: { stream: 'stdout', text: 'compiling\n' } }, id); + emit({ kind: 'output', payload: { stream: 'stderr', text: 'warning: careful\n' } }, id); + emit({ kind: 'scope-end', payload: { status, logPath: relative, ...(summary ? { summary } : {}) } }, id); + emit({ kind: 'producer-finished', payload: { status } }, id); + return path.join(reporter.root, relative); + } + + test('a passing command prints one header and keeps its output in the log', async () => { + const reporter = start(); + command(reporter, 'pod install', '$ pod install\n[stdout] compiling\n'); + await reporter.flush(); + + expect(out).toMatch(/^✔ pod install \([^)]+\)\n$/); + expect(err).toBe(''); + }); + + test('a failed command prints its header, the end of its log, and where the whole log is', async () => { + const reporter = start(); + const file = command(reporter, 'xcodebuild', '$ xcodebuild\n[stdout] compiling\n[stderr] error: boom\n', { + status: 'failed', + summary: 'exit code 65', + }); + await reporter.flush(); + + const [header, ...detail] = err.split('\n'); + expect(header).toMatch(/^✖ xcodebuild \([^)]+\): exit code 65$/); + expect(detail).toEqual([' $ xcodebuild', ' [stdout] compiling', ' [stderr] error: boom', ` Full log: ${file}`, '']); + expect(out).toBe(''); + }); + + test('a failed command with a long log shows the first line and the last two hundred lines', async () => { + const reporter = start(); + const log = ['$ build', ...Array.from({ length: 300 }, (_, index) => `[stdout] line ${index}`)].join('\n') + '\n'; + command(reporter, 'build', log, { status: 'failed' }); + await reporter.flush(); + + const shown = err.split('\n'); + expect(shown.slice(1, 4)).toEqual([' $ build', ' earlier output omitted', ' [stdout] line 100']); + expect(shown).toContain(' [stdout] line 299'); + expect(shown).not.toContain(' [stdout] line 99'); + }); + + test('a failed command whose log cannot be read still reports where the log should be', async () => { + const reporter = start(); + const emit = reporter.source('command-missing'); + emit({ kind: 'scope-start', payload: { label: 'missing', category: 'phase' } }, 'command-missing'); + emit({ kind: 'scope-end', payload: { status: 'failed', logPath: 'commands/missing.log' } }, 'command-missing'); + emit({ kind: 'producer-finished', payload: { status: 'failed' } }, 'command-missing'); + await reporter.flush(); + + expect(err).toContain(' log unavailable: '); + expect(err).toContain(` Full log: ${path.join(reporter.root, 'commands', 'missing.log')}\n`); + }); + + test('redacts secrets from the log it prints', async () => { + const reporter = start(); + reporter.addSecret('hunter2'); + command(reporter, 'login', '$ login\n[stderr] bad password hunter2\n', { status: 'failed', summary: 'exit code 1' }); + await reporter.flush(); + + expect(err).toContain(' [stderr] bad password [REDACTED]\n'); + expect(err).not.toContain('hunter2'); + }); + + test('a verbose run streams a command and then reports only its header', async () => { + const reporter = start({ verbose: true }); + const file = command(reporter, 'pod install', '$ pod install\n'); + await reporter.flush(); + + const lines = out.split('\n'); + expect(lines.slice(0, 3)).toEqual(['▶ pod install', `[pod install] artifact: ${file}`, '[pod install] stdout: compiling']); + expect(lines[3]).toMatch(/^✔ pod install \([^)]+\)$/); + expect(lines).toHaveLength(5); + expect(err).toBe('[pod install] stderr: warning: careful\n'); + }); + + test('a verbose run names the log of a failed command without repeating what it streamed', async () => { + const reporter = start({ verbose: true }); + const file = command(reporter, 'xcodebuild', '$ xcodebuild\n[stdout] compiling\n', { status: 'failed', summary: 'exit code 65' }); + await reporter.flush(); + + expect(err).toMatch(/^\[xcodebuild\] stderr: warning: careful\n✖ xcodebuild \([^)]+\): exit code 65\n {2}Full log: /); + expect(err.endsWith(` Full log: ${file}\n`)).toBe(true); + expect(err).not.toContain('[stdout] compiling'); + expect(out + err).not.toContain('[command-'); + }); + + test('in GitHub Actions a passing command folds its log into a collapsed group', async () => { + const reporter = start({ mode: 'github-actions' }); + const file = command(reporter, 'pod install', '$ pod install\n[stdout] compiling\n'); + await reporter.flush(); + + const [title, ...body] = out.split('\n'); + expect(title).toMatch(/^::group::✔ pod install \([^)]+\)$/); + expect(body).toEqual(['$ pod install', '[stdout] compiling', `Full log: ${file}`, '::endgroup::', '']); + expect(err).toBe(''); + }); + + test('in GitHub Actions a failed command is expanded where it can be seen, not grouped', async () => { + const reporter = start({ mode: 'github-actions' }); + command(reporter, 'xcodebuild', '$ xcodebuild\n[stderr] error: boom\n', { status: 'failed', summary: 'exit code 65' }); + await reporter.flush(); + + expect(out).toBe(''); + expect(err).toMatch(/^✖ xcodebuild \([^)]+\): exit code 65\n {2}\$ xcodebuild\n {2}\[stderr\] error: boom\n {2}Full log: /); + expect(err).not.toContain('::group::'); + }); + + test('a collapsed group keeps the first line and the last thousand lines of a long log', async () => { + const reporter = start({ mode: 'github-actions' }); + const log = ['$ build', ...Array.from({ length: 1200 }, (_, index) => `[stdout] line ${index}`)].join('\n') + '\n'; + command(reporter, 'build', log); + await reporter.flush(); + + const lines = out.split('\n'); + expect(lines.slice(1, 4)).toEqual(['$ build', 'earlier output omitted', '[stdout] line 200']); + expect(lines.at(-4)).toBe('[stdout] line 1199'); + expect(lines).not.toContain('[stdout] line 199'); + }); + + test('a verbose run in GitHub Actions is live and ungrouped', async () => { + const reporter = start({ mode: 'github-actions', verbose: true }); + command(reporter, 'pod install', '$ pod install\n'); + await reporter.flush(); + + expect(out).toContain('[pod install] stdout: compiling\n'); + expect(out).not.toContain('::group::'); + }); + + test('output cannot issue workflow commands, so it needs no stop-commands wrapper', async () => { + const reporter = start({ mode: 'github-actions' }); + command(reporter, 'pod install', '$ pod install\n::warning::from the log\n ##[error]also from the log\n'); + reporter.emit({ kind: 'diagnostic', payload: { level: 'info', message: '::add-mask::value' } }); + reporter.emit({ kind: 'output', payload: { stream: 'stderr', text: '::set-env name=A::B\n' } }); + await reporter.flush(); + + expect(out).toContain(': :add-mask::value\n'); + expect(out).toContain(': :warning::from the log\n'); + expect(out).toContain(' # #[error]also from the log\n'); + expect(err).toBe(': :set-env name=A::B\n'); + expect(out + err).not.toContain('##['); + // Only the group that the reporter wrote itself remains a command. + const commands = (out + err).split('\n').filter((line) => line.trimStart().startsWith('::')); + expect(commands).toHaveLength(2); + expect(commands[0]).toMatch(/^::group::/); + expect(commands[1]).toBe('::endgroup::'); + }); + + test('a finished run has no stop-commands wrappers, doubled line breaks, or command identifiers', async () => { + const reporter = start({ mode: 'github-actions', verbose: true }); + command(reporter, 'pod install', '$ pod install\n[stdout] compiling\n'); + command(reporter, 'xcodebuild', '$ xcodebuild\n[stderr] error: boom\n', { status: 'failed', summary: 'exit code 65' }); + reporter.emit({ kind: 'diagnostic', payload: { level: 'warning', message: 'careful' } }); + await reporter.finish(); + + expect(out + err).not.toContain('::stop-commands::'); + expect(out + err).not.toContain('\n\n'); + expect(out + err).not.toContain('[command-'); + }); + + test('a failure is annotated once in GitHub Actions and titled for the source that reported it', async () => { + const reporter = start({ mode: 'github-actions' }); + const error = new Error('workload claimed'); + reporter.fail(error); + reporter.fail(error); + const emit = reporter.source('command-install'); + emit({ kind: 'scope-start', payload: { label: 'pod install', category: 'phase' } }, 'command-install'); + emit( + { kind: 'diagnostic', payload: { level: 'error', message: 'exit 1', failure: reporter.failure(new Error('exit 1')) } }, + 'command-install', + ); + await reporter.flush(); + + const annotations = err.split('\n').filter((line) => line.startsWith('::error')); + expect(annotations).toHaveLength(2); + expect(annotations[0]).toMatch(/^::error title=desktop%3A infrastructure::Error: workload claimed/); + expect(annotations[1]).toMatch(/^::error title=pod install%3A infrastructure::Error: exit 1/); + expect(err).toContain('error: workload claimed\n'); + expect(err).toContain('[pod install] error: exit 1\n'); + }); + + test('a plain run reports problems and retries but not routine progress', async () => { + const reporter = start({ kind: 'tests' }); + const failure = reporter.failure(new Error('expected 1'), 'test'); + const begin = (testId: string, attempt: number) => reporter.emit({ kind: 'test-start', payload: { testId, title: testId, attempt } }); + const finish = (testId: string, attempt: number, status: 'passed' | 'failed' | 'skipped', extra: object = {}) => + reporter.emit({ kind: 'test-result', payload: { testId, title: testId, attempt, status, durationMs: 1, ...extra } }); + begin('one', 0); + finish('one', 0, 'passed'); + begin('two', 0); + finish('two', 0, 'failed', { reason: 'expected 1', failure }); + begin('two', 1); + finish('two', 1, 'passed'); + finish('three', 0, 'skipped', { reason: 'not on this platform' }); + await reporter.flush(); + + expect(out).toBe('FAILED two: expected 1\nTEST two (attempt 2)\nPASSED two\n'); + expect(err).toMatch(/^Error: expected 1/); + }); + + test('a verbose run reports every test', async () => { + const reporter = start({ kind: 'tests', verbose: true }); + reporter.emit({ kind: 'test-start', payload: { testId: 'one', title: 'one', attempt: 0 } }); + reporter.emit({ kind: 'test-result', payload: { testId: 'one', title: 'one', attempt: 0, status: 'passed', durationMs: 1 } }); + reporter.emit({ + kind: 'test-result', + payload: { testId: 'three', title: 'three', attempt: 0, status: 'skipped', durationMs: 0, reason: 'not on this platform' }, + }); + await reporter.flush(); + + expect(out).toBe('TEST one (attempt 1)\nPASSED one\nSKIPPED three: not on this platform\n'); + }); + + test('a service line names the service and endpoint but no identifier', async () => { + const reporter = start(); + reporter.emit({ kind: 'service-ready', payload: { message: 'listening on 8081' } }, 'metro', { service: 'metro', endpoint: 'macos' }); + await reporter.flush(); + + expect(out).toBe('[metro macos] READY listening on 8081\n'); + }); + + test('a lifecycle run with no tests ends with one status line', async () => { + const reporter = start(); + await reporter.finish(); + + expect(out).toBe(`[desktop] PASSED: lifecycle (no tests); ${path.join(reporter.root, 'run.json')}\n`); + }); +}); diff --git a/packages/agentic/desktop-test/src/reporting/collector.ts b/packages/agentic/desktop-test/src/reporting/collector.ts index 719a281cc6..c6060d58a1 100644 --- a/packages/agentic/desktop-test/src/reporting/collector.ts +++ b/packages/agentic/desktop-test/src/reporting/collector.ts @@ -2,8 +2,8 @@ import fs from 'node:fs'; import path from 'node:path'; import { randomUUID } from 'node:crypto'; import { Writable } from 'node:stream'; -import { createReporter } from '@rnx-kit/reporter'; +import { formatOperationHeader, neutralizeWorkflowCommands, readLogExcerpt, singleLine, trimTrailingLineBreaks } from './presentation.js'; import { createEventEmitter, DesktopReportedError, @@ -13,7 +13,7 @@ import { serializeReportError, validateDesktopEvent, } from './protocol.js'; -import type { CollectedDesktopEvent, EventContext, EventEmitter, FailureKind, ReportError, ReportFailure } from './protocol.js'; +import type { CollectedDesktopEvent, EventContext, EventEmitter, FailureKind, ReportError, ReportFailure, TestStatus } from './protocol.js'; import { describeDesktopSelection, describeDesktopUnsupported, @@ -29,6 +29,14 @@ import type { DesktopReportingConfig } from '../config.js'; import { resolveDesktopReporterMode } from '@fluentui-react-native/desktop-driver/cli/reporter'; import type { ResolvedDesktopReporterMode } from '@fluentui-react-native/desktop-driver/cli/reporter'; +const ROOT_SOURCE = 'supervisor'; +// A failed operation shows the end of its log expanded. A passing one only collapses a longer excerpt into an Actions group. +const expandedExcerpt = { lines: 200, bytes: 64 * 1024 }; +const collapsedExcerpt = { lines: 1000, bytes: 256 * 1024 }; +// These results are progress detail; the final counts account for them. +const quietTestStatuses: ReadonlySet = new Set(['passed', 'skipped', 'unsupported']); +const sourceSuffix = /-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + export type ReportOutput = Pick; export type DesktopRunReporterOptions = DesktopReportingConfig & { artifactsRoot: string; @@ -67,8 +75,10 @@ export class DesktopRunReporter { private readonly sources = new Map(); private readonly scopes = new Map< string, - { label: string; category: string; finished: boolean; context?: EventContext; workId?: string } + { label: string; category: string; finished: boolean; sourceId: string; startedAt: number; context?: EventContext; workId?: string } >(); + // Commands name their own scope, so the label is the most useful name for the lines that command's output produces. + private readonly sourceNames = new Map(); private readonly work = new Map(); private readonly tests = new Map(); private readonly testWork = new Map(); @@ -87,7 +97,6 @@ export class DesktopRunReporter { private readonly errors: ReportOutput; private readonly reporterMode: ResolvedDesktopReporterMode; private readonly structured: boolean; - private readonly logger: ReturnType; private queue = Promise.resolve(); private arrivalSequence = 0; private journalBytes = 0; @@ -121,22 +130,7 @@ export class DesktopRunReporter { this.output = options.output ?? process.stdout; this.errors = options.errorOutput ?? process.stderr; for (const secret of options.secrets ?? []) this.addSecret(secret); - this.emit = createEventEmitter(this.runId, 'supervisor', (event) => this.accept(event)); - this.logger = createReporter({ - name: 'desktop-test', - prefix: {}, - output: { - log: (message) => { - if (!this.structured) this.enqueue(() => this.writeRaw(this.output, message + '\n'), Buffer.byteLength(message)); - }, - warn: (message) => { - if (!this.structured) this.enqueue(() => this.writeRaw(this.errors, message + '\n'), Buffer.byteLength(message)); - }, - error: (message) => { - if (!this.structured) this.enqueue(() => this.writeRaw(this.errors, message + '\n'), Buffer.byteLength(message)); - }, - }, - }); + this.emit = createEventEmitter(this.runId, ROOT_SOURCE, (event) => this.accept(event)); fs.writeFileSync(path.join(this.root, 'events.jsonl'), '', { flag: 'wx', mode: 0o600 }); this.artifacts.set('events.jsonl', 'Versioned event journal'); this.artifacts.set('run.json', 'Canonical run result'); @@ -157,7 +151,7 @@ export class DesktopRunReporter { } source(sourceId: string): EventEmitter { - if (sourceId === 'supervisor' || this.sources.has(sourceId)) throw new Error(`Report source "${sourceId}" is already registered.`); + if (sourceId === ROOT_SOURCE || this.sources.has(sourceId)) throw new Error(`Report source "${sourceId}" is already registered.`); this.sources.set(sourceId, { sequence: 0, finished: false }); return createEventEmitter(this.runId, sourceId, (event) => this.accept(event)); } @@ -202,7 +196,8 @@ export class DesktopRunReporter { } catch (reportingError) { const reportingFailure = this.failure(reportingError, 'reporting'); this.failures.set(reportingFailure.failureId, reportingFailure); - this.logger.error( + this.print( + this.errors, `Desktop reporting failed: ${reportingFailure.error.message}. Original failure: ${failure.error.message}. Reports: ${this.root}`, ); } @@ -281,7 +276,7 @@ export class DesktopRunReporter { } catch (error) { const failure = this.failure(error, 'reporting'); this.failures.set(failure.failureId, failure); - this.logger.error(`Cannot persist desktop run snapshot: ${failure.error.message}`); + this.print(this.errors, `Cannot persist desktop run snapshot: ${failure.error.message}`); } }); } @@ -296,7 +291,15 @@ export class DesktopRunReporter { if (this.scopes.has(event.scopeId)) throw new Error(`Scope ${event.scopeId} was started twice.`); const work = event.parentScopeId ? this.work.get(event.parentScopeId) : undefined; if (work) work.status = 'incomplete'; - this.scopes.set(event.scopeId, { ...event.payload, finished: false, context: event.context, workId: work?.workId }); + this.scopes.set(event.scopeId, { + ...event.payload, + finished: false, + sourceId: event.sourceId, + startedAt: Date.parse(event.timestamp), + context: event.context, + workId: work?.workId, + }); + if (event.scopeId === event.sourceId) this.sourceNames.set(event.sourceId, singleLine(event.payload.label, 48)); } else if (event.kind === 'scope-end') { const scope = this.scopes.get(event.scopeId); if (!scope || scope.finished) throw new Error(`Scope ${event.scopeId} has no outstanding start.`); @@ -356,54 +359,57 @@ export class DesktopRunReporter { } return; } - const prefix = `[${event.sourceId}${event.context?.endpoint ? ` ${event.context.endpoint}` : ''}]`; + const verbose = this.options.verbose === true; + const tag = this.tag(event); const failure = 'failure' in event.payload ? event.payload.failure : undefined; const firstFailure = !failure || !this.presentedFailures.has(failure.failureId); if (failure) this.presentedFailures.add(failure.failureId); switch (event.kind) { case 'scope-start': - this.logger.log(`${prefix} START ${event.payload.label}`); + if (verbose) this.print(this.output, `▶ ${singleLine(event.payload.label, 200)}`); break; - case 'scope-end': { - const scope = this.scopes.get(event.scopeId)!; - if (event.payload.logPath && (event.payload.status === 'failed' || this.options.verbose || scope.category === 'worker')) { - this.enqueue(() => this.replay(scope.label, event.payload.logPath!, event.payload.status === 'failed'), 0); - } - this.logger.log(`${prefix} ${event.payload.status.toUpperCase()} ${scope.label}`); + case 'scope-end': + this.presentScopeEnd(event, verbose); break; - } case 'service-ready': - this.logger.log(`${prefix} READY ${event.payload.message}`); + this.print(this.output, `${tag}READY ${event.payload.message}`); break; case 'output': - if (event.payload.stream === 'stderr' || this.options.verbose) - this.logger.warn(`${prefix} ${event.payload.stream}: ${event.payload.text.replace(/\n$/, '')}`); + // A command's output is kept in its log and replayed if it fails, so it is live only in verbose mode. + if ((verbose || (event.payload.stream === 'stderr' && event.sourceId === ROOT_SOURCE)) && event.payload.text.trim()) { + this.print( + event.payload.stream === 'stderr' ? this.errors : this.output, + tag ? `${tag}${event.payload.stream}: ${event.payload.text}` : event.payload.text, + ); + } break; case 'diagnostic': if (!firstFailure) break; - if (event.payload.level === 'info') this.logger.log(`${prefix} ${event.payload.message}`); - else this.logger.warn(`${prefix} ${event.payload.level}: ${event.payload.message}`); + if (event.payload.level === 'info') this.print(this.output, `${tag}${event.payload.message}`); + else this.print(this.errors, `${tag}${event.payload.level}: ${event.payload.message}`); break; case 'service-failed': - if (firstFailure) this.logger.error(`${prefix} FAILED ${event.payload.failure.error.message}`); + if (firstFailure) this.print(this.errors, `${tag}FAILED ${event.payload.failure.error.message}`); break; case 'test-start': { - // Producers may encode attempts sparsely (WDIO combines spec-file and test retries), so label the observed ordinal. - const ordinal = this.tests.get(event.payload.testId)!.attempts.findIndex(({ attempt }) => attempt === event.payload.attempt) + 1; - this.logger.log(`${prefix} TEST ${event.payload.title} (attempt ${ordinal})`); + const ordinal = this.attemptOrdinal(event.payload.testId, event.payload.attempt); + // The first attempt is progress detail; a retry is worth marking. + if (verbose || ordinal > 1) this.print(this.output, `${tag}TEST ${event.payload.title} (attempt ${ordinal})`); break; } - case 'test-result': - this.logger.log( - `${prefix} ${event.payload.status.toUpperCase()} ${event.payload.title}${event.payload.reason ? `: ${event.payload.reason}` : ''}`, - ); + case 'test-result': { + const { status, title, reason } = event.payload; + const ordinal = this.attemptOrdinal(event.payload.testId, event.payload.attempt); + if (verbose || !quietTestStatuses.has(status) || ordinal > 1) + this.print(this.output, `${tag}${status.toUpperCase()} ${title}${reason ? `: ${reason}` : ''}`); // Reported, never failed: a declaration may cover a path this attempt did not take. - if (event.payload.input?.unused) - this.logger.log(`${prefix} declared input this attempt did not deliver: ${event.payload.input.unused.join(', ')}`); - if (failure && firstFailure) this.logger.error(`${prefix} ${errorDiagnostic(failure.error)}`); + if (verbose && event.payload.input?.unused) + this.print(this.output, `${tag}declared input this attempt did not deliver: ${event.payload.input.unused.join(', ')}`); + if (failure && firstFailure) this.print(this.errors, `${tag}${errorDiagnostic(failure.error)}`); break; + } case 'artifact': - this.logger.log(`${prefix} artifact: ${path.join(this.root, event.payload.path)}`); + if (verbose) this.print(this.output, `${tag}artifact: ${path.join(this.root, event.payload.path)}`); break; } if ('failure' in event.payload && event.payload.failure) { @@ -414,7 +420,7 @@ export class DesktopRunReporter { () => this.write( this.errors, - `::error title=${escapeProperty(`${event.sourceId}: ${failure.kind}`)}::${escapeData(errorDiagnostic(failure.error))}\n`, + `::error title=${escapeProperty(`${this.sourceName(event.sourceId) ?? 'desktop'}: ${failure.kind}`)}::${escapeData(errorDiagnostic(failure.error))}\n`, ), 0, ); @@ -474,36 +480,74 @@ export class DesktopRunReporter { }); } - private async writeRaw(output: ReportOutput, text: string): Promise { - if (!this.actions) { - await this.write(output, text); - return; - } - const token = randomUUID(); - await this.write(output, `::stop-commands::${token}\n`); - try { - await this.write(output, text.endsWith('\n') ? text : text + '\n'); - } finally { - await this.write(output, `::${token}::\n`); + private print(output: ReportOutput, text: string): void { + if (this.structured) return; + this.enqueue(() => this.writeLine(output, text), Buffer.byteLength(text)); + } + + // Everything the reporter prints, except the commands it builds itself, passes through here so output cannot issue workflow commands. + private writeLine(output: ReportOutput, text: string): Promise { + return this.write(output, neutralizeWorkflowCommands(trimTrailingLineBreaks(text)) + '\n'); + } + + private sourceName(sourceId: string): string | undefined { + if (sourceId === ROOT_SOURCE) return undefined; + return ( + this.sourceNames.get(sourceId) ?? + sourceId + .split('/') + .map((segment) => segment.replace(sourceSuffix, '')) + .join('/') + ); + } + + /** `[name endpoint] ` for lines from a command or producer, and for supervisor lines about a service or endpoint. */ + private tag(event: CollectedDesktopEvent): string { + const name = this.sourceName(event.sourceId) ?? event.context?.service; + const endpoint = event.context?.endpoint; + const parts = [name, endpoint && !name?.includes(endpoint) ? endpoint : undefined].filter(Boolean); + return parts.length ? `[${parts.join(' ')}] ` : ''; + } + + // Producers may encode attempts sparsely (WDIO combines spec-file and test retries), so label the observed ordinal. + private attemptOrdinal(testId: string, attempt: number): number { + return this.tests.get(testId)!.attempts.findIndex((candidate) => candidate.attempt === attempt) + 1; + } + + private presentScopeEnd(event: Extract, verbose: boolean): void { + const scope = this.scopes.get(event.scopeId)!; + const { status, logPath, summary } = event.payload; + const attention = status === 'failed' || status === 'cancelled'; + // A scope with a log or a summary is an operation and always reports its outcome. Other scopes are progress detail. + if (logPath === undefined && summary === undefined && !attention && !verbose) return; + const header = formatOperationHeader({ + status, + label: scope.label, + durationMs: Date.parse(event.timestamp) - scope.startedAt, + detail: summary, + }); + const output = attention ? this.errors : this.output; + const file = logPath === undefined ? undefined : this.artifactPath(logPath); + if (file !== undefined && !verbose && (attention || this.actions)) { + this.enqueue(() => this.writeOperation(output, header, file, attention), 0); + } else { + // Verbose mode streamed the output as it arrived, and a plain run keeps a passing operation's output in its log. + this.print(output, attention && file !== undefined ? `${header}\n Full log: ${file}` : header); } } - private async replay(label: string, relative: string, failed: boolean): Promise { - const output = failed ? this.errors : this.output; - await this.write(output, this.actions ? `::group::${escapeData(label)}\n` : `[desktop] BEGIN ${label}\n`); - const token = randomUUID(); + /** A failed operation prints the end of its log under its header. In Actions a passing one folds it into a collapsed group. */ + private async writeOperation(output: ReportOutput, header: string, file: string, expanded: boolean): Promise { + let lines: string[]; try { - if (this.actions) await this.write(output, `::stop-commands::${token}\n`); - try { - for await (const chunk of fs.createReadStream(this.artifactPath(relative), { encoding: 'utf8' })) - await this.write(output, this.redact(chunk)); - await this.write(output, '\n'); - } finally { - if (this.actions) await this.write(output, `::${token}::\n`); - } - } finally { - await this.write(output, this.actions ? '::endgroup::\n' : `[desktop] END ${label}\n`); + const excerpt = await readLogExcerpt(file, expanded ? expandedExcerpt : collapsedExcerpt, 'earlier output omitted'); + lines = excerpt.map((line) => this.redact(line)); + } catch (error) { + lines = [`log unavailable: ${error instanceof Error ? error.message : String(error)}`]; } + lines.push(`Full log: ${file}`); + if (expanded) await this.writeLine(output, [header, ...lines.map((line) => ` ${line}`)].join('\n')); + else await this.write(output, `::group::${escapeData(header)}\n${neutralizeWorkflowCommands(lines.join('\n'))}\n::endgroup::\n`); } finish(): Promise { @@ -525,7 +569,7 @@ export class DesktopRunReporter { optional: this.options.optional, incomplete: [...this.scopes.values()].some(({ finished }) => !finished) || - [...this.sources.entries()].some(([id, source]) => id !== 'supervisor' && !source.finished), + [...this.sources.entries()].some(([id, source]) => id !== ROOT_SOURCE && !source.finished), tests: [...this.tests.values()], work: [...this.work.values()], failures: [...this.failures.values()], @@ -602,19 +646,20 @@ export class DesktopRunReporter { }; let result = persist(); const c = result.counts; - this.logger.log( - `[desktop] ${result.status.toUpperCase()}: ${c.selected} selected, ${c.executed} executed, ${c.passed} passed, ${c.failed} failed, ${c.skipped} skipped, ${describeDesktopUnsupported(c)}, ${c.incomplete} incomplete, ${c.retries} retries, ${c.flaky} flaky; ${result.work.filter(({ status }) => status === 'not-run').length} groups not run; ${path.join(this.root, 'run.json')}`, + const runResult = path.join(this.root, 'run.json'); + this.print( + this.output, + result.kind === 'lifecycle' && c.selected === 0 + ? `[desktop] ${result.status.toUpperCase()}: lifecycle (no tests); ${runResult}` + : `[desktop] ${result.status.toUpperCase()}: ${c.selected} selected, ${c.executed} executed, ${c.passed} passed, ${c.failed} failed, ${c.skipped} skipped, ${describeDesktopUnsupported(c)}, ${c.incomplete} incomplete, ${c.retries} retries, ${c.flaky} flaky; ${result.work.filter(({ status }) => status === 'not-run').length} groups not run; ${runResult}`, ); const selection = describeDesktopSelection(result); - if (selection) this.logger.log(`[desktop] ${selection}`); + if (selection) this.print(this.output, `[desktop] ${selection}`); await this.queue; result = persist(); if (this.presentationFailed && !this.structured) { try { - await this.writeRaw( - this.errors, - `[desktop] FAILED: presentation could not be flushed; authoritative result: ${path.join(this.root, 'run.json')}\n`, - ); + await this.writeLine(this.errors, `[desktop] FAILED: presentation could not be flushed; authoritative result: ${runResult}`); } catch (error) { const failure = this.failure(error, 'reporting'); this.failures.set(failure.failureId, failure); @@ -679,7 +724,7 @@ export class DesktopRunReporter { result = persist(); } try { - await this.writeRaw(this.errors, `[desktop] Structured output stopped: ${outputError.message}\n`); + await this.writeLine(this.errors, `[desktop] Structured output stopped: ${outputError.message}`); } catch { // A closed stderr cannot carry the reporting diagnostic. } diff --git a/packages/agentic/desktop-test/src/reporting/index.ts b/packages/agentic/desktop-test/src/reporting/index.ts index 4765f332aa..b5a82caa40 100644 --- a/packages/agentic/desktop-test/src/reporting/index.ts +++ b/packages/agentic/desktop-test/src/reporting/index.ts @@ -2,6 +2,8 @@ export { DesktopRunReporter } from './collector.js'; export type { DesktopReportWriter, DesktopRunReporterOptions, ReportOutput } from './collector.js'; export { createDesktopReportSession } from './session.js'; export type { DesktopReportSession } from './session.js'; +export { runDesktopOperation } from './operation.js'; +export type { DesktopOperation, DesktopOperationOptions } from './operation.js'; export { createEventEmitter, DesktopReportedError, serializeReportError, validateDesktopEvent } from './protocol.js'; export type { DesktopEvent, diff --git a/packages/agentic/desktop-test/src/reporting/operation.test.ts b/packages/agentic/desktop-test/src/reporting/operation.test.ts new file mode 100644 index 0000000000..13a439475d --- /dev/null +++ b/packages/agentic/desktop-test/src/reporting/operation.test.ts @@ -0,0 +1,224 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { DesktopRunReporter } from './collector.js'; +import { runDesktopOperation } from './operation.js'; +import type { EventData, EventEmitter } from './protocol.js'; + +describe('runDesktopOperation', () => { + type Emitted = { event: EventData; scopeId?: string; context?: unknown }; + + function recorder(failOn?: EventData['kind']) { + const emitted: Emitted[] = []; + const emit: EventEmitter = (event, scopeId, context) => { + if (event.kind === failOn) throw new Error(`cannot report ${event.kind}`); + emitted.push({ event, scopeId, context }); + }; + return { writer: { emit }, emitted }; + } + + test('reports a passing operation with its summary and returns the result', async () => { + const { writer, emitted } = recorder(); + + const result = await runDesktopOperation(writer, { label: 'Render stories', scopeId: 'render' }, (operation) => { + operation.summarize('2 stories rendered'); + return 42; + }); + + expect(result).toBe(42); + expect(emitted).toEqual([ + { event: { kind: 'scope-start', payload: { label: 'Render stories', category: 'phase' } }, scopeId: 'render', context: undefined }, + { event: { kind: 'scope-end', payload: { status: 'passed', summary: '2 stories rendered' } }, scopeId: 'render', context: undefined }, + ]); + }); + + test('ends a passing operation with an empty summary so that the reporter prints its outcome', async () => { + const { writer, emitted } = recorder(); + + await runDesktopOperation(writer, { label: 'Step', scopeId: 'step' }, () => undefined); + + expect(emitted.at(-1)?.event).toEqual({ kind: 'scope-end', payload: { status: 'passed', summary: '' } }); + }); + + test('reports a failure with the first line of its message and throws it again', async () => { + const { writer, emitted } = recorder(); + const error = new Error('build failed\nsecond line'); + + await expect( + runDesktopOperation(writer, { label: 'Step', scopeId: 'step' }, () => { + throw error; + }), + ).rejects.toBe(error); + + expect(emitted.at(-1)?.event).toEqual({ kind: 'scope-end', payload: { status: 'failed', summary: 'build failed' } }); + }); + + test('prefers the summary of the work over the message of its failure', async () => { + const { writer, emitted } = recorder(); + + await expect( + runDesktopOperation(writer, { label: 'Step', scopeId: 'step' }, async (operation) => { + operation.summarize('1 of 3 failed'); + throw new Error('details'); + }), + ).rejects.toThrow('details'); + + expect(emitted.at(-1)?.event).toEqual({ kind: 'scope-end', payload: { status: 'failed', summary: '1 of 3 failed' } }); + }); + + test('describes a thrown value that is not an error', async () => { + const { writer, emitted } = recorder(); + + await expect( + runDesktopOperation(writer, { label: 'Step', scopeId: 'step' }, () => { + throw 'plain text'; + }), + ).rejects.toBe('plain text'); + + expect(emitted.at(-1)?.event).toEqual({ kind: 'scope-end', payload: { status: 'failed', summary: 'plain text' } }); + }); + + test('throws the failure of the work, not the failure to report it', async () => { + const { writer } = recorder('scope-end'); + const error = new Error('real failure'); + + await expect( + runDesktopOperation(writer, { label: 'Step' }, () => { + throw error; + }), + ).rejects.toBe(error); + }); + + test('surfaces a failure to report a passing operation', async () => { + const { writer } = recorder('scope-end'); + + await expect(runDesktopOperation(writer, { label: 'Step' }, () => 1)).rejects.toThrow('cannot report scope-end'); + }); + + test('records detail as one stdout line per non-empty line, in the scope and context of the operation', async () => { + const { writer, emitted } = recorder(); + const context = { service: 'storybook', endpoint: 'macos' } as const; + + await runDesktopOperation(writer, { label: 'Step', scopeId: 'step', context, category: 'worker' }, (operation) => { + operation.log('one\n\ntwo \r\n'); + }); + + expect(emitted.map(({ event }) => event.kind)).toEqual(['scope-start', 'output', 'output', 'scope-end']); + expect(emitted[0].event).toEqual({ kind: 'scope-start', payload: { label: 'Step', category: 'worker' } }); + expect(emitted.slice(1, 3).map(({ event, scopeId, context: eventContext }) => ({ event, scopeId, eventContext }))).toEqual([ + { event: { kind: 'output', payload: { stream: 'stdout', text: 'one\n' } }, scopeId: 'step', eventContext: context }, + { event: { kind: 'output', payload: { stream: 'stdout', text: 'two\n' } }, scopeId: 'step', eventContext: context }, + ]); + expect(emitted[3].context).toBe(context); + }); + + test('keeps a summary to one line', async () => { + const { writer, emitted } = recorder(); + + await runDesktopOperation(writer, { label: 'Step' }, (operation) => { + operation.summarize('first\nsecond'); + }); + + expect(emitted.at(-1)?.event).toMatchObject({ payload: { summary: 'first second' } }); + }); + + test('runs the work when nothing is listening', async () => { + const result = await runDesktopOperation(undefined, { label: 'Step' }, (operation) => { + operation.summarize('ignored'); + operation.log('ignored'); + return 'done'; + }); + + expect(result).toBe('done'); + }); + + describe('with a reporter', () => { + let artifactsRoot: string; + let out: string; + let err: string; + + beforeEach(() => { + artifactsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-operation-')); + out = ''; + err = ''; + }); + + afterEach(() => { + fs.rmSync(artifactsRoot, { force: true, recursive: true }); + }); + + function start(verbose: boolean, kind: 'tests' | 'lifecycle' = 'tests') { + return new DesktopRunReporter({ + artifactsRoot, + kind, + output: { write: (chunk: string) => ((out += chunk), true) }, + errorOutput: { write: (chunk: string) => ((err += chunk), true) }, + reporterMode: 'plain', + summary: false, + verbose, + }); + } + + test('a passing operation prints one header line and keeps its detail out of the output', async () => { + const reporter = start(false); + + await runDesktopOperation(reporter, { label: 'Render stories' }, (operation) => { + operation.log('rendered one'); + operation.summarize('1 stories rendered'); + }); + await reporter.flush(); + + expect(out).toMatch(/^✔ Render stories \([^)]+\): 1 stories rendered\n$/); + expect(err).toBe(''); + }); + + test('verbose output shows the detail between the start and the header', async () => { + const reporter = start(true); + + await runDesktopOperation(reporter, { label: 'Render stories' }, (operation) => { + operation.log('rendered one'); + }); + await reporter.flush(); + + expect(out).toMatch(/^▶ Render stories\nrendered one\n✔ Render stories \([^)]+\)\n$/); + }); + + test('a failing operation prints its header on the error stream', async () => { + const reporter = start(false); + + await expect( + runDesktopOperation(reporter, { label: 'Render stories' }, () => { + throw new Error('1 of 3 stories failed to render'); + }), + ).rejects.toThrow('1 of 3 stories failed to render'); + await reporter.flush(); + + expect(out).toBe(''); + expect(err).toMatch(/^✖ Render stories \([^)]+\): 1 of 3 stories failed to render\n$/); + }); + + test('redacts secrets from the summary', async () => { + const reporter = start(false); + reporter.addSecret('hunter2'); + + await runDesktopOperation(reporter, { label: 'Sign in' }, (operation) => { + operation.summarize('token hunter2 accepted'); + }); + await reporter.flush(); + + expect(out).toContain('token [REDACTED] accepted'); + expect(out).not.toContain('hunter2'); + }); + + test('operations can repeat on one reporter, and each one completes', async () => { + const reporter = start(false, 'lifecycle'); + + await runDesktopOperation(reporter, { label: 'Step' }, () => undefined); + await runDesktopOperation(reporter, { label: 'Step' }, () => undefined); + + await expect(reporter.finish()).resolves.toMatchObject({ status: 'passed' }); + expect(out.match(/✔ Step/g)).toHaveLength(2); + }); + }); +}); diff --git a/packages/agentic/desktop-test/src/reporting/operation.ts b/packages/agentic/desktop-test/src/reporting/operation.ts new file mode 100644 index 0000000000..560291e2d8 --- /dev/null +++ b/packages/agentic/desktop-test/src/reporting/operation.ts @@ -0,0 +1,65 @@ +import { randomUUID } from 'node:crypto'; + +import { singleLine } from './presentation.js'; +import type { EventContext, EventEmitter } from './protocol.js'; + +export type DesktopOperationOptions = { + /** What the operation does, such as `Render stories`. It is the label of the outcome header. */ + label: string; + category?: 'service' | 'worker' | 'phase'; + /** A scope can start only once, so the default is unique. */ + scopeId?: string; + context?: EventContext; +}; + +export type DesktopOperation = { + /** Sets the outcome the header shows, such as `30 of 101 story tests executed`. A later call replaces an earlier one. */ + summarize(summary: string): void; + /** Records a line of detail that is shown live only in verbose mode. */ + log(text: string): void; +}; + +/** + * Runs `work` as a reported operation. When it ends the reporter prints one outcome header, `✔ label (4.2s)` or + * `✖ label (4.2s): reason`. A failure from `work` is reported and then thrown again. Detail that has no log of its own + * goes through `log`, so that only verbose runs show it. + */ +export async function runDesktopOperation( + writer: { readonly emit: EventEmitter } | undefined, + options: DesktopOperationOptions, + work: (operation: DesktopOperation) => T | Promise, +): Promise { + const scopeId = options.scopeId ?? `operation-${randomUUID()}`; + let summary: string | undefined; + const operation: DesktopOperation = { + summarize: (text) => { + summary = singleLine(text, 400); + }, + log: (text) => { + for (const line of text.split('\n')) { + const trimmed = line.trimEnd(); + if (trimmed) writer?.emit({ kind: 'output', payload: { stream: 'stdout', text: `${trimmed}\n` } }, scopeId, options.context); + } + }, + }; + const end = (status: 'passed' | 'failed', outcome: string) => + writer?.emit({ kind: 'scope-end', payload: { status, summary: outcome } }, scopeId, options.context); + writer?.emit({ kind: 'scope-start', payload: { label: options.label, category: options.category ?? 'phase' } }, scopeId, options.context); + let result: T; + try { + result = await work(operation); + } catch (error) { + try { + end('failed', summary ?? singleLine(describeError(error).split('\n', 1)[0], 400)); + } catch { + // The failure of the work is the error to surface, not a failure to report it. + } + throw error; + } + end('passed', summary ?? ''); + return result; +} + +function describeError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/agentic/desktop-test/src/reporting/presentation.test.ts b/packages/agentic/desktop-test/src/reporting/presentation.test.ts new file mode 100644 index 0000000000..28c16529ae --- /dev/null +++ b/packages/agentic/desktop-test/src/reporting/presentation.test.ts @@ -0,0 +1,215 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { + formatDuration, + formatOperationHeader, + neutralizeWorkflowCommands, + readLogExcerpt, + singleLine, + trimTrailingLineBreaks, +} from './presentation.js'; + +describe('formatDuration', () => { + test.each([ + [0, '0ms'], + [850, '850ms'], + [999, '999ms'], + [999.6, '1.0s'], + [4234, '4.2s'], + [9950, '10s'], + [42_000, '42s'], + [59_500, '1m 00s'], + [123_000, '2m 03s'], + [3_599_500, '1h 00m'], + [3_723_000, '1h 02m'], + [-5, '0ms'], + [Number.NaN, '0ms'], + [Number.POSITIVE_INFINITY, '0ms'], + ])('%p is %s', (milliseconds, expected) => { + expect(formatDuration(milliseconds)).toBe(expected); + }); +}); + +describe('singleLine', () => { + test('replaces every kind of line separator with a space', () => { + expect(singleLine('one\ntwo\r\nthree\u0085four\u2028five\u2029six')).toBe('one two three four five six'); + }); + + test('trims trailing whitespace but keeps tabs and escape sequences', () => { + expect(singleLine('a\tb \n')).toBe('a\tb'); + expect(singleLine('\u001b[31mred\u001b[0m')).toBe('\u001b[31mred\u001b[0m'); + }); + + test('clips long text with an ellipsis and never leaves half of a surrogate pair', () => { + expect(singleLine('abc', 3)).toBe('abc'); + expect(singleLine('abcdef', 3)).toBe('abc…'); + expect(singleLine('ab\u{1F600}', 3)).toBe('ab…'); + expect(singleLine('ab\u{1F600}', 4)).toBe('ab\u{1F600}'); + }); +}); + +describe('neutralizeWorkflowCommands', () => { + test.each([ + ['plain text', 'plain text'], + ['', ''], + ['text ::error::not at the start', 'text ::error::not at the start'], + ['::error::boom', ': :error::boom'], + ['::stop-commands::token', ': :stop-commands::token'], + [' \t::warning::indented', ' \t: :warning::indented'], + [':::triple', ': ::triple'], + ['abc ##[group]title', 'abc # #[group]title'], + ['##[error]boom', '# #[error]boom'], + ])('%j becomes %j', (text, expected) => { + expect(neutralizeWorkflowCommands(text)).toBe(expected); + }); + + test('looks past every character the runner trims, and only those', () => { + expect(neutralizeWorkflowCommands('\u00a0::x')).toBe('\u00a0: :x'); + expect(neutralizeWorkflowCommands('\u0085::x')).toBe('\u0085: :x'); + expect(neutralizeWorkflowCommands('\u2028::x')).toBe('\u2028: :x'); + expect(neutralizeWorkflowCommands('\u3000::x')).toBe('\u3000: :x'); + expect(neutralizeWorkflowCommands('\u200b::x')).toBe('\u200b::x'); + }); + + test('guards each line, where the runner splits on both CR and LF', () => { + expect(neutralizeWorkflowCommands('::a::\nok\n::b::\n')).toBe(': :a::\nok\n: :b::\n'); + expect(neutralizeWorkflowCommands('safe\r::error::boom')).toBe('safe\r: :error::boom'); + expect(neutralizeWorkflowCommands('safe\r\n::error::boom\r\n')).toBe('safe\r\n: :error::boom\r\n'); + }); + + test('is idempotent', () => { + for (const text of ['::error::boom', ' ::x::\n##[group]y\n::z::', 'ok']) { + const once = neutralizeWorkflowCommands(text); + expect(neutralizeWorkflowCommands(once)).toBe(once); + } + }); +}); + +describe('trimTrailingLineBreaks', () => { + test.each([ + ['a\n\n', 'a'], + ['a\r\n', 'a'], + ['a\nb', 'a\nb'], + ['a \n', 'a '], + ['\n', ''], + ['', ''], + ])('%j becomes %j', (text, expected) => { + expect(trimTrailingLineBreaks(text)).toBe(expected); + }); +}); + +describe('formatOperationHeader', () => { + test('shows the status, label, duration, and detail', () => { + expect(formatOperationHeader({ status: 'passed', label: 'pod install', durationMs: 42_000 })).toBe('✔ pod install (42s)'); + expect(formatOperationHeader({ status: 'passed', label: 'Render stories', durationMs: 2, detail: '2 stories rendered' })).toBe( + '✔ Render stories (2ms): 2 stories rendered', + ); + expect(formatOperationHeader({ status: 'failed', label: 'rnx-cli build', durationMs: 243_000, detail: 'exit code 65' })).toBe( + '✖ rnx-cli build (4m 03s): exit code 65', + ); + }); + + test('omits the duration when it is unknown and the detail when it is empty', () => { + expect(formatOperationHeader({ status: 'passed', label: 'Step' })).toBe('✔ Step'); + expect(formatOperationHeader({ status: 'passed', label: 'Step', detail: '' })).toBe('✔ Step'); + expect(formatOperationHeader({ status: 'failed', label: 'Step' })).toBe('✖ Step'); + }); + + test('names the status of an operation that did not pass or fail', () => { + expect(formatOperationHeader({ status: 'skipped', label: 'Step' })).toBe('○ Step: skipped'); + expect(formatOperationHeader({ status: 'unsupported', label: 'Step' })).toBe('○ Step: unsupported'); + expect(formatOperationHeader({ status: 'cancelled', label: 'Step' })).toBe('⚠ Step: cancelled'); + expect(formatOperationHeader({ status: 'skipped', label: 'Step', detail: 'no devices' })).toBe('○ Step: no devices'); + }); + + test('keeps the header on one line', () => { + expect(formatOperationHeader({ status: 'failed', label: 'two\nlines', detail: 'first\nsecond' })).toBe('✖ two lines: first second'); + expect(formatOperationHeader({ status: 'passed', label: 'x'.repeat(300) })).toBe(`✔ ${'x'.repeat(200)}…`); + }); +}); + +describe('readLogExcerpt', () => { + let directory: string; + const note = 'earlier output omitted'; + const numbered = (count: number) => Array.from({ length: count }, (_, index) => `line-${String(index).padStart(2, '0')}`); + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-excerpt-')); + }); + + afterEach(() => { + fs.rmSync(directory, { force: true, recursive: true }); + }); + + function log(content: string): string { + const file = path.join(directory, 'command.log'); + fs.writeFileSync(file, content); + return file; + } + + test('returns a log within the limits whole', async () => { + await expect(readLogExcerpt(log('$ cmd\n[stdout] one\n[stderr] two\n'), { lines: 10, bytes: 1024 }, note)).resolves.toEqual([ + '$ cmd', + '[stdout] one', + '[stderr] two', + ]); + }); + + test('returns a last line that has no line break, keeps carriage returns, and handles an empty log', async () => { + await expect(readLogExcerpt(log('a\r\nb'), { lines: 10, bytes: 1024 }, note)).resolves.toEqual(['a\r', 'b']); + await expect(readLogExcerpt(log(''), { lines: 10, bytes: 1024 }, note)).resolves.toEqual([]); + }); + + test('keeps the first line, a note, and the last lines when there are too many', async () => { + const lines = numbered(20); + + await expect(readLogExcerpt(log(lines.join('\n') + '\n'), { lines: 3, bytes: 1024 * 1024 }, note)).resolves.toEqual([ + 'line-00', + note, + 'line-17', + 'line-18', + 'line-19', + ]); + }); + + test('starts the last bytes at a line boundary', async () => { + // Each line is 8 bytes, so a 50 byte window begins inside line-13 and the excerpt begins with the next whole line. + const file = log(numbered(20).join('\n') + '\n'); + + await expect(readLogExcerpt(file, { lines: 1000, bytes: 50 }, note)).resolves.toEqual(['line-00', note, ...numbered(20).slice(14)]); + // A window that begins exactly where a line does keeps that line. + await expect(readLogExcerpt(file, { lines: 1000, bytes: 56 }, note)).resolves.toEqual(['line-00', note, ...numbered(20).slice(13)]); + }); + + test('never starts the excerpt inside a multi-byte character', async () => { + const excerpt = await readLogExcerpt(log('first\n€€€€\nlast\n'), { lines: 1000, bytes: 8 }, note); + + expect(excerpt).toEqual(['first', note, 'last']); + }); + + test('shows the end of a single line that is larger than the window', async () => { + const excerpt = await readLogExcerpt(log('x'.repeat(10_000)), { lines: 10, bytes: 100 }, note); + + expect(excerpt).toHaveLength(3); + expect(excerpt[0]).toBe(`${'x'.repeat(4096)}…`); + expect(excerpt[1]).toBe(note); + expect(excerpt[2]).toMatch(/^…x{100,101}$/); + }); + + test('shows the end of a newline-terminated line that is larger than the window', async () => { + const excerpt = await readLogExcerpt(log(`$ cmd\n${'x'.repeat(10_000)}\n`), { lines: 10, bytes: 100 }, note); + + expect(excerpt).toHaveLength(3); + expect(excerpt[0]).toBe('$ cmd'); + expect(excerpt[1]).toBe(note); + expect(excerpt[2]).toMatch(/^…x{98,100}$/); + }); + + test('rejects when the log cannot be read', async () => { + await expect(readLogExcerpt(path.join(directory, 'missing.log'), { lines: 10, bytes: 1024 }, note)).rejects.toMatchObject({ + code: 'ENOENT', + }); + }); +}); diff --git a/packages/agentic/desktop-test/src/reporting/presentation.ts b/packages/agentic/desktop-test/src/reporting/presentation.ts new file mode 100644 index 0000000000..3cd228eafb --- /dev/null +++ b/packages/agentic/desktop-test/src/reporting/presentation.ts @@ -0,0 +1,164 @@ +import fs from 'node:fs/promises'; +import type { FileHandle } from 'node:fs/promises'; + +import type { EventData } from './protocol.js'; + +export type DesktopScopeStatus = Extract['payload']['status']; + +const symbols: Record = { passed: '✔', failed: '✖', skipped: '○', unsupported: '○', cancelled: '⚠' }; +const headBytes = 4096; + +/** Formats a duration for an operation header: `850ms`, `4.2s`, `42s`, `2m 03s`, or `1h 02m`. */ +export function formatDuration(milliseconds: number): string { + const whole = Math.round(Number.isFinite(milliseconds) ? Math.max(0, milliseconds) : 0); + if (whole < 1000) return `${whole}ms`; + const tenths = Math.round(whole / 100); + if (tenths < 100) return `${Math.floor(tenths / 10)}.${tenths % 10}s`; + const seconds = Math.round(whole / 1000); + if (seconds < 60) return `${seconds}s`; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) return `${minutes}m ${String(seconds % 60).padStart(2, '0')}s`; + return `${Math.floor(minutes / 60)}h ${String(minutes % 60).padStart(2, '0')}m`; +} + +// The Actions runner trims these (.NET `Char.IsWhiteSpace`) before it looks for `::`. +function isRunnerWhitespace(code: number): boolean { + return ( + (code >= 0x09 && code <= 0x0d) || + code === 0x20 || + code === 0x85 || + code === 0xa0 || + code === 0x1680 || + (code >= 0x2000 && code <= 0x200a) || + code === 0x2028 || + code === 0x2029 || + code === 0x202f || + code === 0x205f || + code === 0x3000 + ); +} + +function neutralizeLine(line: string): string { + let start = 0; + while (start < line.length && isRunnerWhitespace(line.charCodeAt(start))) start++; + const guarded = line.startsWith('::', start) ? `${line.slice(0, start + 1)} ${line.slice(start + 1)}` : line; + return guarded.replaceAll('##[', '# #['); +} + +/** + * Defuses text the Actions runner would read as a workflow command: `::` after the leading whitespace of a line (the + * runner splits lines on CR and LF) and `##[` anywhere. Everything the reporter prints that it did not build as a + * command passes through this, so child output needs no `::stop-commands::` wrapper. Only text that would have been + * a command changes. + */ +export function neutralizeWorkflowCommands(text: string): string { + if (!text.includes('::') && !text.includes('##[')) return text; + let result = ''; + let start = 0; + for (let index = 0; index <= text.length; index++) { + const code = index < text.length ? text.charCodeAt(index) : 0x0a; + if (code !== 0x0a && code !== 0x0d) continue; + result += neutralizeLine(text.slice(start, index)) + (index < text.length ? text[index] : ''); + start = index + 1; + } + return result; +} + +/** Drops trailing line breaks so one printed message is one block of lines. */ +export function trimTrailingLineBreaks(text: string): string { + let end = text.length; + while (end > 0 && (text[end - 1] === '\n' || text[end - 1] === '\r')) end--; + return end === text.length ? text : text.slice(0, end); +} + +/** + * Collapses text to one line: line separators become spaces and text longer than `maxLength` is clipped with an + * ellipsis. Escape sequences and tabs are kept so colored child output survives. + */ +export function singleLine(text: string, maxLength = 4096): string { + const limit = Math.min(text.length, maxLength); + let line = ''; + for (let index = 0; index < limit; index++) { + const code = text.charCodeAt(index); + line += code === 0x0a || code === 0x0d || code === 0x85 || code === 0x2028 || code === 0x2029 ? ' ' : text[index]; + } + const clipped = text.length > limit; + const last = line.charCodeAt(line.length - 1); + if (clipped && last >= 0xd800 && last <= 0xdbff) line = line.slice(0, -1); + return line.trimEnd() + (clipped ? '…' : ''); +} + +export type DesktopOperationHeader = { + status: DesktopScopeStatus; + label: string; + durationMs?: number; + /** The outcome in a few words. Skipped, unsupported, and cancelled operations name their status when it is omitted. */ + detail?: string; +}; + +/** The one-line outcome of an operation, such as `✔ pod install (42s)` or `✖ rnx-cli build (4m 03s): exit code 65`. */ +export function formatOperationHeader({ status, label, durationMs, detail }: DesktopOperationHeader): string { + const duration = durationMs === undefined ? '' : ` (${formatDuration(durationMs)})`; + const outcome = detail ?? (status === 'passed' || status === 'failed' ? undefined : status); + return `${symbols[status]} ${singleLine(label, 200)}${duration}${outcome ? `: ${singleLine(outcome, 400)}` : ''}`; +} + +export type DesktopLogExcerptLimits = { lines: number; bytes: number }; + +async function readRange(handle: FileHandle, position: number, length: number): Promise { + const buffer = Buffer.alloc(length); + let filled = 0; + while (filled < length) { + const { bytesRead } = await handle.read(buffer, filled, length - filled, position + filled); + if (bytesRead === 0) break; + filled += bytesRead; + } + return filled === length ? buffer : buffer.subarray(0, filled); +} + +/** + * Reads the lines of a command log to show. A log within `limits` comes back whole. A larger one keeps its first line, + * which names the command, then `omittedNote`, then the last lines that fit. + */ +export async function readLogExcerpt(file: string, limits: DesktopLogExcerptLimits, omittedNote: string): Promise { + const handle = await fs.open(file, 'r'); + try { + const { size } = await handle.stat(); + const windowStart = Math.max(0, size - limits.bytes); + // The byte before the window shows whether the window begins on a line boundary. + const from = Math.max(0, windowStart - 1); + const window = await readRange(handle, from, size - from); + const partial = windowStart > 0; + let offset = 0; + let cut = false; + if (partial) { + offset = 1; + if (window[0] !== 0x0a) { + const newline = window.indexOf(0x0a, 1); + // A single line can fill the whole window; its end is more useful than nothing. + cut = newline < 0 || newline === window.length - 1; + offset = cut ? 0 : newline + 1; + } + } + const lines = window.toString('utf8', offset).split('\n'); + if (cut) { + let start = 0; + while (lines[0][start] === '\uFFFD') start++; + lines[0] = `…${lines[0].slice(start)}`; + } + if (lines.at(-1) === '') lines.pop(); + const capped = lines.length > limits.lines; + const tail = capped ? lines.slice(lines.length - Math.max(0, limits.lines)) : lines; + if (!partial && !capped) return tail; + const head = await readRange(handle, 0, Math.min(size, headBytes)); + const newline = head.indexOf(0x0a); + let first = head.toString('utf8', 0, newline < 0 ? head.length : newline); + if (newline < 0 && size > head.length) { + while (first.endsWith('\uFFFD')) first = first.slice(0, -1); + first += '…'; + } + return [first, omittedNote, ...tail]; + } finally { + await handle.close(); + } +} diff --git a/packages/agentic/desktop-test/src/reporting/protocol.test.ts b/packages/agentic/desktop-test/src/reporting/protocol.test.ts index 315aa96b2f..786a29748c 100644 --- a/packages/agentic/desktop-test/src/reporting/protocol.test.ts +++ b/packages/agentic/desktop-test/src/reporting/protocol.test.ts @@ -60,6 +60,12 @@ describe('declared gaps and selection events', () => { expect(() => validateDesktopEvent(event('selection', { excluded: { platform: 1, scope: 0, input: 12 } }))).not.toThrow(); }); + test('accepts a scope end summary, even an empty one, and rejects one that is not text', () => { + expect(() => validateDesktopEvent(event('scope-end', { status: 'passed', summary: '30 of 101 story tests executed' }))).not.toThrow(); + expect(() => validateDesktopEvent(event('scope-end', { status: 'passed', summary: '' }))).not.toThrow(); + expect(() => validateDesktopEvent(event('scope-end', { status: 'passed', summary: 3 }))).toThrow('summary must be text'); + }); + test('accepts an optional idempotency key and rejects an empty one', () => { const excluded = { platform: 1, scope: 0, input: 0 }; expect(() => validateDesktopEvent(event('selection', { excluded, key: 'wdio-group' }))).not.toThrow(); diff --git a/packages/agentic/desktop-test/src/reporting/protocol.ts b/packages/agentic/desktop-test/src/reporting/protocol.ts index 14b2754831..46bfa5b69f 100644 --- a/packages/agentic/desktop-test/src/reporting/protocol.ts +++ b/packages/agentic/desktop-test/src/reporting/protocol.ts @@ -45,7 +45,18 @@ export type EventContext = { export type EventData = | { kind: 'work-selected'; payload: { workId: string; label: string } } | { kind: 'scope-start'; payload: { label: string; category: 'service' | 'worker' | 'phase' } } - | { kind: 'scope-end'; payload: { status: 'passed' | 'failed' | 'cancelled' | 'skipped' | 'unsupported'; logPath?: string } } + | { + kind: 'scope-end'; + payload: { + status: 'passed' | 'failed' | 'cancelled' | 'skipped' | 'unsupported'; + logPath?: string; + /** + * One line naming the outcome, such as `exit code 1` or `30 of 101 story tests executed`. A scope that ends with a + * summary, even an empty one, or with a log is an operation: the reporter always prints its outcome. + */ + summary?: string; + }; + } | { kind: 'service-ready'; payload: { message: string } } | { kind: 'service-failed'; payload: { failure: ReportFailure } } | { kind: 'diagnostic'; payload: { level: 'info' | 'warning' | 'error'; message: string; failure?: ReportFailure } } @@ -88,7 +99,7 @@ export const reportStatuses: readonly TestStatus[] = ['passed', 'failed', 'skipp const payloadFields = { 'work-selected': ['workId', 'label'], 'scope-start': ['label', 'category'], - 'scope-end': ['status', 'logPath'], + 'scope-end': ['status', 'logPath', 'summary'], 'service-ready': ['message'], 'service-failed': ['failure'], diagnostic: ['level', 'message', 'failure'], @@ -165,6 +176,7 @@ export function validateDesktopEvent(value: unknown): asserts value is DesktopEv break; case 'scope-end': if (payload.logPath !== undefined) requireText(payload.logPath, 'logPath'); + if (payload.summary !== undefined && typeof payload.summary !== 'string') throw new TypeError('summary must be text.'); member(payload.status, ['passed', 'failed', 'cancelled', 'skipped', 'unsupported']); break; case 'producer-finished': diff --git a/packages/agentic/desktop-test/src/reporting/structured.test.ts b/packages/agentic/desktop-test/src/reporting/structured.test.ts index 74a2c7f3d8..23b29dd175 100644 --- a/packages/agentic/desktop-test/src/reporting/structured.test.ts +++ b/packages/agentic/desktop-test/src/reporting/structured.test.ts @@ -20,6 +20,14 @@ describe('structured desktop events', () => { expect(includeDesktopEvent(event, 'milestones', 'service')).toBe(true); }); + test('a scope end projects its summary only when there is one', () => { + const summarized = makeEvent({ kind: 'scope-end', payload: { status: 'passed', summary: '30 of 101 story tests executed' } }); + const plain = makeEvent({ kind: 'scope-end', payload: { status: 'passed', summary: '' } }); + + expect(projectDesktopEvent(summarized).payload).toEqual({ status: 'passed', summary: '30 of 101 story tests executed' }); + expect(projectDesktopEvent(plain).payload).toEqual({ status: 'passed' }); + }); + test('exclusion counts are progress detail and project as plain numbers', () => { const event = makeEvent({ kind: 'selection', payload: { excluded: { platform: 1, scope: 0, input: 2 } } }); diff --git a/packages/agentic/desktop-test/src/reporting/structured.ts b/packages/agentic/desktop-test/src/reporting/structured.ts index 544be9ebc9..ed3ded7ee0 100644 --- a/packages/agentic/desktop-test/src/reporting/structured.ts +++ b/packages/agentic/desktop-test/src/reporting/structured.ts @@ -61,6 +61,7 @@ export function projectDesktopEvent(event: CollectedDesktopEvent): DesktopEventR case 'scope-end': payload.status = event.payload.status; if (event.payload.logPath) payload.logPath = clip(event.payload.logPath); + if (event.payload.summary) payload.summary = clip(event.payload.summary); break; case 'service-ready': payload.message = clip(event.payload.message); diff --git a/packages/agentic/desktop-test/src/services.ts b/packages/agentic/desktop-test/src/services.ts index 5569bcf267..af66045d7a 100644 --- a/packages/agentic/desktop-test/src/services.ts +++ b/packages/agentic/desktop-test/src/services.ts @@ -91,7 +91,12 @@ export class DesktopServiceScope { }); let running: RunningDesktopCommand; try { - running = this.runner.start({ ...command, category: 'service', context: { ...command.context, service: id } }); + running = this.runner.start({ + ...command, + label: command.label ?? id, + category: 'service', + context: { ...command.context, service: id }, + }); resource.dispose = () => running.stop({ failed: true }); } catch (error) { resource.record.readiness = 'failed'; diff --git a/packages/agentic/storybook-desktop/config/storybook-control.cjs b/packages/agentic/storybook-desktop/config/storybook-control.cjs index 88531b9463..1072c3002b 100644 --- a/packages/agentic/storybook-desktop/config/storybook-control.cjs +++ b/packages/agentic/storybook-desktop/config/storybook-control.cjs @@ -12,6 +12,7 @@ const initialRenderMilliseconds = Number(process.env.STORYBOOK_SMOKE_INITIAL_REN const controller = new AbortController(); let report; let runner; +let runOperation; function parseSmokePhase(args) { if (args.length === 0) { @@ -61,77 +62,88 @@ async function smoke(traversal) { throw new Error(`Unsupported Storybook smoke mode "${smokeMode}".`); } if (smokePhase !== 'tests') { - const index = await request('/index.json'); - const entries = Object.values(index.entries || {}).filter(({ type }) => type === 'story'); - if (entries.length === 0) { - throw new Error('The Storybook index did not contain any stories.'); - } + await runOperation(report.reporter, { label: 'Render stories' }, (operation) => renderStories(traversal, operation)); + } +} + +async function renderStories(traversal, operation) { + const index = await request('/index.json'); + const entries = Object.values(index.entries || {}).filter(({ type }) => type === 'story'); + if (entries.length === 0) { + throw new Error('The Storybook index did not contain any stories.'); + } - const settleMilliseconds = Number(process.env.STORYBOOK_SMOKE_SETTLE_MS) || 0; - const failFast = process.env.STORYBOOK_SMOKE_FAIL_FAST === '1'; - const failures = []; - for (const { id } of entries) - report.reporter.emit({ kind: 'work-selected', payload: { workId: `navigation:${id}`, label: `Navigate ${id}` } }); + const settleMilliseconds = Number(process.env.STORYBOOK_SMOKE_SETTLE_MS) || 0; + const failFast = process.env.STORYBOOK_SMOKE_FAIL_FAST === '1'; + const failures = []; + for (const { id } of entries) + report.reporter.emit({ kind: 'work-selected', payload: { workId: `navigation:${id}`, label: `Navigate ${id}` } }); - for (const [entryIndex, { id }] of entries.entries()) { - const started = Date.now(); - const testId = `navigation:${id}`; - const title = `Navigate ${id}`; - const context = { story: id }; - report.reporter.emit({ kind: 'scope-start', payload: { label: title, category: 'phase' } }, testId, context, testId); - report.reporter.emit({ kind: 'test-start', payload: { testId, title, attempt: 0 } }, testId, context); - try { - if (traversal) await traversal.select(id, entryIndex === 0 ? initialRenderMilliseconds : 30_000); - else await selectStory(id); - if (settleMilliseconds > 0) { - await new Promise((resolve) => setTimeout(resolve, settleMilliseconds)); - } - report.reporter.emit( - { kind: 'test-result', payload: { testId, title, attempt: 0, status: 'passed', durationMs: Date.now() - started } }, - testId, - context, - ); - report.reporter.emit({ kind: 'scope-end', payload: { status: 'passed' } }, testId); - } catch (error) { - failures.push(error); - report.reporter.emit( - { - kind: 'test-result', - payload: { - testId, - title, - attempt: 0, - status: 'failed', - durationMs: Date.now() - started, - failure: report.reporter.failure(error, 'test'), - }, + let attempted = 0; + for (const [entryIndex, { id }] of entries.entries()) { + attempted += 1; + const started = Date.now(); + const testId = `navigation:${id}`; + const title = `Navigate ${id}`; + const context = { story: id }; + report.reporter.emit({ kind: 'scope-start', payload: { label: title, category: 'phase' } }, testId, context, testId); + report.reporter.emit({ kind: 'test-start', payload: { testId, title, attempt: 0 } }, testId, context); + try { + if (traversal) await traversal.select(id, entryIndex === 0 ? initialRenderMilliseconds : 30_000); + else await selectStory(id); + if (settleMilliseconds > 0) { + await new Promise((resolve) => setTimeout(resolve, settleMilliseconds)); + } + report.reporter.emit( + { kind: 'test-result', payload: { testId, title, attempt: 0, status: 'passed', durationMs: Date.now() - started } }, + testId, + context, + ); + report.reporter.emit({ kind: 'scope-end', payload: { status: 'passed' } }, testId); + } catch (error) { + failures.push(error); + report.reporter.emit( + { + kind: 'test-result', + payload: { + testId, + title, + attempt: 0, + status: 'failed', + durationMs: Date.now() - started, + failure: report.reporter.failure(error, 'test'), }, - testId, - context, - ); - report.reporter.emit({ kind: 'scope-end', payload: { status: 'failed' } }, testId); - if (failFast) { - break; - } + }, + testId, + context, + ); + report.reporter.emit({ kind: 'scope-end', payload: { status: 'failed' } }, testId); + if (failFast) { + break; } } + } - if (failures.length > 0) { - throw new AggregateError(failures, `${failures.length} of ${entries.length} stories failed to render.`); - } - report.reporter.emit({ kind: 'diagnostic', payload: { level: 'info', message: `Rendered ${entries.length} stories.` } }); + if (failures.length > 0) { + const notAttempted = entries.length - attempted; + operation.summarize( + `${failures.length} of ${entries.length} stories failed to render${notAttempted > 0 ? ` (stopped early; ${notAttempted} not attempted)` : ''}`, + ); + throw new AggregateError(failures, `${failures.length} of ${entries.length} stories failed to render.`); } + operation.summarize(`${entries.length} ${entries.length === 1 ? 'story' : 'stories'} rendered`); } async function runAuthoredTests() { const manifest = readDriverManifest(); const smokeTestsUrl = pathToFileURL(path.join(__dirname, '..', 'lib', 'cli', 'smokeTests.js')).href; const configUrl = pathToFileURL(path.join(__dirname, '..', 'lib', 'config', 'index.js')).href; - const { formatDesktopStorybookSmokeTestSummary, runDesktopStorybookSmokeTests } = await import(smokeTestsUrl); + const { runDesktopStorybookSmokeTests } = await import(smokeTestsUrl); const { makeDesktopStorybookConfig } = await import(configUrl); // The driver instance's lane is a per-run choice the CLI resolved, so it travels as an option, not as configuration. const { input, ...wdio } = manifest.wdio ?? {}; - const result = await runDesktopStorybookSmokeTests({ + // The story tests report their own outcome header, with the counts, through `report`. + await runDesktopStorybookSmokeTests({ config: makeDesktopStorybookConfig({ projectRoot: process.cwd(), wdio }), ...(input ? { input } : {}), ...(manifest.scope ? { scope: manifest.scope } : {}), @@ -150,7 +162,6 @@ async function runAuthoredTests() { }, }, }); - report.reporter.emit({ kind: 'diagnostic', payload: { level: 'info', message: formatDesktopStorybookSmokeTestSummary(result) } }); } function readDriverManifest() { @@ -174,10 +185,11 @@ function readDriverManifest() { } async function main() { - const [{ createDesktopReportSession }, { NodeDesktopCommandRunner }] = await Promise.all([ + const [{ createDesktopReportSession, runDesktopOperation }, { NodeDesktopCommandRunner }] = await Promise.all([ import('@fluentui-react-native/desktop-test/reporting'), import('@fluentui-react-native/desktop-test/commands'), ]); + runOperation = runDesktopOperation; report = await createDesktopReportSession({ artifactsRoot: path.join(process.cwd(), 'artifacts', 'desktop'), kind: 'tests' }); runner = new NodeDesktopCommandRunner({ reporter: report.reporter, environment: report.environment }); report.onStop?.(() => controller.abort(new Error('Storybook control interrupted.'))); diff --git a/packages/agentic/storybook-desktop/jest.config.cjs b/packages/agentic/storybook-desktop/jest.config.cjs index 8f56587ad6..b28d09e48f 100644 --- a/packages/agentic/storybook-desktop/jest.config.cjs +++ b/packages/agentic/storybook-desktop/jest.config.cjs @@ -2,7 +2,7 @@ const config = require('@fluentui-react-native/scripts/jest-config'); module.exports = { ...config, - transformIgnorePatterns: ['node_modules[/\\\\](?!.*(?:\\.mjs$|@rnx-kit(?:[/\\\\]reporter|-reporter-npm-)))'], + transformIgnorePatterns: ['node_modules[/\\\\](?!.*\\.mjs$)'], moduleNameMapper: { ...config.moduleNameMapper, '^(\\.{1,2}/.*)\\.js$': '$1', diff --git a/packages/agentic/storybook-desktop/references/commands.md b/packages/agentic/storybook-desktop/references/commands.md index 34ea5326e9..8e63384484 100644 --- a/packages/agentic/storybook-desktop/references/commands.md +++ b/packages/agentic/storybook-desktop/references/commands.md @@ -20,6 +20,16 @@ Global output options are `--reporter auto|plain|github-actions|json|ndjson`, only). The mode precedence is CLI flag, `DESKTOP_REPORTER`, config, then `auto`. Global options may appear before or after a subcommand. +Plain and GitHub Actions output reports each step as one header with its +outcome and duration, and shows the end of a step's log only when it fails. The +`Render stories` and `Story tests` steps of a smoke run carry the counts in +their headers, such as `209 stories rendered` and +`101 of 101 story tests executed: 101 passed, 0 failed, 0 skipped`. A run that +executes no story tests says `no story tests executed (0 of N)`. Pass +`-v`/`--verbose` to stream command output live, with start markers and log +paths. The console format is described in +`@fluentui-react-native/desktop-test/references/reporting.md`. + | Command | Purpose | | --------------------------------------------- | --------------------------------------------------- | | `manifest --` | Validate static story-plan extraction | diff --git a/packages/agentic/storybook-desktop/references/platforms/macos.md b/packages/agentic/storybook-desktop/references/platforms/macos.md index 4a6c03c5e8..d386448752 100644 --- a/packages/agentic/storybook-desktop/references/platforms/macos.md +++ b/packages/agentic/storybook-desktop/references/platforms/macos.md @@ -14,8 +14,9 @@ project. Run the application bundle before `prep` when CocoaPods must embed a JavaScript bundle or other resource. CocoaPods includes only resources present during `pod install`. Use `yarn storybook prep --macos`, `bundle --macos`, -`build --macos`, and `smoke --macos`; use `--verbose` to capture actionable -native diagnostics. Do not patch generated Pod sources or workspace files. +`build --macos`, and `smoke --macos`. A failed step prints the end of its log and +the log's path; use `--verbose` to stream native diagnostics live instead. Do +not patch generated Pod sources or workspace files. Storybook app diagnostics and native permissions are documented in the consumer's agent entry point and the repository permissions reference. Only diff --git a/packages/agentic/storybook-desktop/references/troubleshooting.md b/packages/agentic/storybook-desktop/references/troubleshooting.md index c05b700dcb..e84466c0f0 100644 --- a/packages/agentic/storybook-desktop/references/troubleshooting.md +++ b/packages/agentic/storybook-desktop/references/troubleshooting.md @@ -21,6 +21,16 @@ Triage by failure boundary: 6. **Evidence/cleanup:** inspect the complete canonical run directory; preserve the primary failure if capture or cleanup also fails. +Console output summarizes each step as one header, such as `✔ pod install (42s)` +or `✖ rnx-cli build (4m 03s): exit code 65`. A failed step prints the end of its +log and a `Full log:` path; the complete output of every step is under the run's +`commands/` directory, and `--verbose` streams it live. In a smoke run, read the +`Story tests` header: `N of M story tests executed` says how many of the M +tests reached ran, and the counts after the colon account for the rest as +skipped or unsupported. A run that stops at its first failure adds +`(stopped early; K of S stories not run)`. `no story tests executed (0 of M)` +means none ran, even if nothing failed. + For `driver --agent`, a readiness timeout means the channel service did not publish the connection: inspect the `storybook-channel` log. Publication needs the verified native helper and an owner-only storage root. A macOS lease diff --git a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts index 531ed97042..d8eeec0efc 100644 --- a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts +++ b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts @@ -7,12 +7,14 @@ import { validateDesktopCaseOptions } from '@fluentui-react-native/desktop-drive import { DesktopServiceScope } from '@fluentui-react-native/desktop-test'; import type { ResolvedDesktopConfig, ResolvedDesktopTarget } from '@fluentui-react-native/desktop-test'; import { selectDesktopRenderer } from '@fluentui-react-native/desktop-test/config'; +import type { DesktopReportWriter } from '@fluentui-react-native/desktop-test/reporting'; import { STORYBOOK_SMOKE_MODE } from '../config/commands'; import { DesktopStorybookConfig, makeDesktopStorybookConfig } from '../config/makeDesktopStorybookConfig'; import { FURN_STORYBOOK_BUNDLE_IDENTIFIER, FURN_STORYBOOK_INSTANCE_ID } from '../config/instance'; import { FURN_STORYBOOK_PLATFORM } from '../config/platforms'; import { desktopStorybookTargetId } from '../driver/driverManifest'; +import * as wdioTests from '../testing/runWdioTests'; import type { DesktopCommandRunner, PreparedDesktopCommand, RunningDesktopCommand } from './commandRunner'; import { createDesktopStorybookCommand } from './createDesktopStorybookCommand'; import { @@ -978,7 +980,7 @@ describe('DesktopStorybookCli', () => { `${serverUrl}/select-story-sync/first--story`, `${serverUrl}/select-story-sync/second--story`, ]); - expect(output.at(-1)).toBe('Rendered 2 stories.\n'); + expect(output.at(-1)).toBe('2 stories rendered\n'); expect(runner.background[1].args).toEqual(['start', '--no-interactive', '--port', String(cli.instance.metroPort + 1)]); expect(runner.foreground.at(-1)?.command).toBe('stop-storybook'); expect(runner.stopped).toBe(2); @@ -1066,7 +1068,7 @@ describe('DesktopStorybookCli', () => { const result = cli.smoke('macos').catch((error: unknown) => error); await jest.advanceTimersByTimeAsync(131_000); expect(await result).toBeUndefined(); - expect(output.write).toHaveBeenCalledWith('Rendered 1 stories.\n'); + expect(output.write).toHaveBeenCalledWith('1 story rendered\n'); expect(runner.stopped).toBe(2); } finally { jest.useRealTimers(); @@ -1273,6 +1275,144 @@ describe('DesktopStorybookCli', () => { platform: 'macos', }); }); + + describe('outcome headers', () => { + const stories = { entries: { first: { id: 'first--story', type: 'story' }, second: { id: 'second--story', type: 'story' } } }; + const connection = { url: 'https://127.0.0.1:1234', target: 'fixture' }; + const quiet = { write: () => true }; + + const fakeReporter = () => { + const emit = jest.fn(); + const reporter = { runId: 'run', root: storybookRoot, emit, failure: jest.fn(), fail: jest.fn() } as unknown as DesktopReportWriter; + return { reporter, emit }; + }; + const events = (emit: jest.Mock) => emit.mock.calls.map(([event]) => event); + const outcomes = (emit: jest.Mock) => events(emit).filter(({ kind, payload }) => kind === 'scope-end' && payload.summary !== undefined); + const details = (emit: jest.Mock) => events(emit).flatMap(({ kind, payload }) => (kind === 'output' ? [payload.text] : [])); + const renderingFetch = + (select: () => Response): typeof globalThis.fetch => + async (input) => + String(input).includes('select-story-sync') ? select() : new Response(JSON.stringify(stories)); + const smokeCli = (fetch: typeof globalThis.fetch, options: object) => + new DesktopStorybookCli( + makeConfig({ + macos: { run: { command: 'launch-storybook' }, smoke: { stop: { command: 'stop-storybook' }, startupTimeoutMs: 100 } }, + }), + { + ...nativeDriverTestOptions, + createStoryManifest: createEmptyStoryManifest, + runner: new RecordingRunner(), + fetch, + isPortAvailable: async () => true, + errorOutput: quiet, + ...options, + }, + ); + const wdioResult = ( + testName: string, + status: wdioTests.WdioStoryTestResult['status'], + gap?: 'unsupported', + ): wdioTests.WdioStoryTestResult => ({ + storyId: 'button--default', + testName, + status, + durationMs: 1, + ...(gap ? { gap } : {}), + }); + const testCli = (options: object) => + new DesktopStorybookCli(makeDesktopStorybookConfig({ projectRoot: storybookRoot, storyPackages: [] }), { + createStoryManifest: storyManifestOf(declaredStoryEntry('button--default', [['adaptive', { requires: ['click'] }]])), + errorOutput: quiet, + ...options, + }); + + afterEach(() => jest.restoreAllMocks()); + + test('summarizes the rendered stories as the outcome of the render step and keeps each story as verbose-only detail', async () => { + const { reporter, emit } = fakeReporter(); + const output = { write: jest.fn() }; + await smokeCli( + renderingFetch(() => new Response('{}')), + { reporter, output }, + ).smoke('macos'); + expect(outcomes(emit)).toEqual([{ kind: 'scope-end', payload: { status: 'passed', summary: '2 stories rendered' } }]); + expect(details(emit)).toEqual(['rendered first--story\n', 'rendered second--story\n']); + expect(output.write).not.toHaveBeenCalledWith(expect.stringContaining('rendered first--story')); + }); + + test('names how many stories failed to render, and that the run stopped early, in the failed outcome', async () => { + const { reporter, emit } = fakeReporter(); + const fetch = renderingFetch(() => new Response(JSON.stringify({ error: 'render failed' }), { status: 500 })); + await expect(smokeCli(fetch, { reporter, output: quiet }).smoke('macos')).rejects.toThrow('first--story'); + expect(outcomes(emit)).toEqual([ + { kind: 'scope-end', payload: { status: 'failed', summary: '1 of 2 stories failed to render (stopped early; 1 not attempted)' } }, + ]); + }); + + test('prints the render outcome as a line, also when it failed, when there is no reporter', async () => { + const output: string[] = []; + const sink = { write: (text: string | Uint8Array) => !!output.push(String(text)) }; + await expect( + smokeCli( + renderingFetch(() => new Response(JSON.stringify({ error: 'render failed' }), { status: 500 })), + { output: sink }, + ).smoke('macos'), + ).rejects.toThrow('first--story'); + expect(output).toContain('1 of 2 stories failed to render (stopped early; 1 not attempted)\n'); + }); + + test.each([false, true])( + 'writes a line for each rendered story only in verbose mode without a reporter (verbose: %s)', + async (verbose) => { + const output: string[] = []; + const sink = { write: (text: string | Uint8Array) => !!output.push(String(text)) }; + await smokeCli( + renderingFetch(() => new Response('{}')), + { output: sink, verbose }, + ).smoke('macos'); + expect(output.includes('rendered first--story\n')).toBe(verbose); + expect(output).toContain('2 stories rendered\n'); + }, + ); + + test('summarizes how many story tests executed and passed as the outcome of the test step', async () => { + const { reporter, emit } = fakeReporter(); + jest + .spyOn(wdioTests, 'runWdioStoryTests') + .mockResolvedValueOnce([wdioResult('adaptive', 'passed'), wdioResult('typing', 'unsupported', 'unsupported')]); + await testCli({ reporter, output: quiet }).test('macos', connection); + expect(outcomes(emit)).toEqual([ + { + kind: 'scope-end', + payload: { + status: 'passed', + summary: '1 of 2 story tests executed: 1 passed, 0 failed, 0 skipped, 1 unsupported (1 declared, 0 quarantined)', + }, + }, + ]); + }); + + test('summarizes the counts a failed test run left, not only the error', async () => { + const { reporter, emit } = fakeReporter(); + const failure = new wdioTests.WdioStoryTestRunError( + [wdioResult('adaptive', 'passed'), wdioResult('typing', 'failed')], + [new Error('typing failed')], + ); + jest.spyOn(wdioTests, 'runWdioStoryTests').mockRejectedValueOnce(failure); + await expect(testCli({ reporter, output: quiet }).test('macos', connection)).rejects.toBe(failure); + expect(outcomes(emit)).toEqual([ + { kind: 'scope-end', payload: { status: 'failed', summary: '2 of 2 story tests executed: 1 passed, 1 failed, 0 skipped' } }, + ]); + }); + + test('prints the test outcome as a line when there is no reporter', async () => { + const output: string[] = []; + const sink = { write: (text: string | Uint8Array) => !!output.push(String(text)) }; + jest.spyOn(wdioTests, 'runWdioStoryTests').mockResolvedValueOnce([wdioResult('adaptive', 'passed')]); + await testCli({ output: sink }).test('macos', connection); + expect(output).toEqual(['1 of 1 story tests executed: 1 passed, 0 failed, 0 skipped\n']); + }); + }); }); describe('createDesktopStorybookCommand', () => { diff --git a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts index 22000c76e8..76546ef8b6 100644 --- a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts +++ b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts @@ -8,7 +8,7 @@ import { resolveDesktopApplication, resolveDesktopTestScope, } from '@fluentui-react-native/desktop-test'; -import type { DesktopReportWriter } from '@fluentui-react-native/desktop-test/reporting'; +import { runDesktopOperation, type DesktopOperation, type DesktopReportWriter } from '@fluentui-react-native/desktop-test/reporting'; import { resolveDesktopScopeDomains } from '@fluentui-react-native/desktop-driver/authoring'; import { formatDesktopReporterValue } from '@fluentui-react-native/desktop-driver/cli/reporter'; import type { ResolvedDesktopReporterMode } from '@fluentui-react-native/desktop-driver/cli/reporter'; @@ -53,8 +53,7 @@ import { import { NodeDesktopCommandRunner, STORYBOOK_VERBOSE } from './commandRunner.js'; import type { DesktopCommandRunner, PreparedDesktopCommand, RunningDesktopCommand } from './commandRunner.js'; import { formatDesktopStorybookSmokeTestSummary, runDesktopStorybookSmokeTests } from './smokeTests.js'; -import { runWdioStoryTests } from '../testing/runWdioTests.js'; -import { describeWdioCounts } from '../testing/laneVerdict.js'; +import { runWdioStoryTests, WdioStoryTestRunError, type WdioStoryTestResult } from '../testing/runWdioTests.js'; import { decideWdioSelection, describeWdioExclusions, listWdioSelection } from '../testing/selection.js'; import type { WdioSelectionContext } from '../testing/selection.js'; import { resolveWdioOptions, type DesktopStorybookWdioOptions } from '../config/wdio.js'; @@ -384,22 +383,52 @@ export class DesktopStorybookCli { if (platform === 'macos' && !options.url) { await this.writeMacOSApplicationLease(desktopStorybookDriverManifestPath(this.config.projectRoot, 'macos'), settings.timeoutMs); } - const results = await runWdioStoryTests( - { - ...settings, - ...activeConnection, - config: this.config, - manifest, - platform, - scope, - errorOutput: this.errorOutput, - report: this.reporter, - signal: this.services.controller.signal, - input: settings.input, - }, - this.runner, - ); - this.output.write(`Ran ${results.length} executable wdio tests (${describeWdioCounts(results)}).\n`); + const summarize = (operation: DesktopOperation, wdio: readonly WdioStoryTestResult[]) => + operation.summarize(formatDesktopStorybookSmokeTestSummary({ tests: [], wdio })); + await this.operation('Story tests', async (operation) => { + try { + const results = await runWdioStoryTests( + { + ...settings, + ...activeConnection, + config: this.config, + manifest, + platform, + scope, + errorOutput: this.errorOutput, + report: this.reporter, + signal: this.services.controller.signal, + input: settings.input, + }, + this.runner, + ); + summarize(operation, results); + } catch (error) { + if (error instanceof WdioStoryTestRunError) summarize(operation, error.results); + throw error; + } + }); + } + + /** + * Runs `work` as one reported operation, which ends with an outcome header. Without a reporter, which only a library + * caller can lack, the summary is written as a line and detail is written only in verbose mode. + */ + private async operation(label: string, work: (operation: DesktopOperation) => T | Promise): Promise { + if (this.reporter) return runDesktopOperation(this.reporter, { label }, work); + let summary: string | undefined; + try { + return await work({ + summarize: (text) => { + summary = text; + }, + log: (text) => { + if (this.verbose) this.output.write(`${text}\n`); + }, + }); + } finally { + if (summary) this.output.write(`${summary}\n`); + } } /** The scope domains and the values this endpoint's target holds, which declared case `scope` selection reads. */ @@ -611,7 +640,15 @@ export class DesktopStorybookCli { let traversalFailure: unknown; try { await this.services.guard( - this.renderEveryStory(serverUrl, smoke.settleMs ?? 0, smoke.initialRenderTimeoutMs ?? smoke.startupTimeoutMs, traversal?.select), + this.operation('Render stories', (operation) => + this.renderEveryStory( + operation, + serverUrl, + smoke.settleMs ?? 0, + smoke.initialRenderTimeoutMs ?? smoke.startupTimeoutMs, + traversal?.select, + ), + ), ); } catch (error) { traversalFailure = error; @@ -640,7 +677,8 @@ export class DesktopStorybookCli { input, }), ); - this.output.write(`${formatDesktopStorybookSmokeTestSummary(result)}\n`); + // With a reporter the story tests operation's header already carries the summary. + if (!this.reporter) this.output.write(`${formatDesktopStorybookSmokeTestSummary(result)}\n`); } } catch (error) { primaryFailure = error; @@ -830,6 +868,7 @@ export class DesktopStorybookCli { } private async renderEveryStory( + operation: DesktopOperation, serverUrl: string, settleMs: number, initialRenderTimeoutMs = 300_000, @@ -846,9 +885,11 @@ export class DesktopStorybookCli { const failures: Error[] = []; for (const { id } of entries) this.reporter?.emit({ kind: 'work-selected', payload: { workId: `navigation:${id}`, label: `Render ${id}` } }); + let attempted = 0; let consecutiveFailures = 0; for (const [entryIndex, { id }] of entries.entries()) { this.services.controller.signal.throwIfAborted(); + attempted += 1; const started = Date.now(); const testId = `navigation:${id}`; this.reporter?.emit({ kind: 'scope-start', payload: { label: `Render ${id}`, category: 'worker' } }, testId, { story: id }, testId); @@ -859,7 +900,7 @@ export class DesktopStorybookCli { if (settleMs > 0) { await delay(settleMs); } - this.output.write(`rendered ${id}\n`); + operation.log(`rendered ${id}`); this.reporter?.emit( { kind: 'test-result', @@ -896,9 +937,13 @@ export class DesktopStorybookCli { } if (failures.length > 0) { + const notAttempted = entries.length - attempted; + operation.summarize( + `${failures.length} of ${entries.length} stories failed to render${notAttempted > 0 ? ` (stopped early; ${notAttempted} not attempted)` : ''}`, + ); throw new AggregateError(failures, `${failures.length} stories failed to render:\n${failures.map(errorMessage).join('\n')}`); } - this.output.write(`Rendered ${entries.length} stories.\n`); + operation.summarize(`${entries.length} ${entries.length === 1 ? 'story' : 'stories'} rendered`); } private async selectStory(serverUrl: string, storyId: string, timeoutMs: number): Promise { diff --git a/packages/agentic/storybook-desktop/src/cli/commandRunner.test.ts b/packages/agentic/storybook-desktop/src/cli/commandRunner.test.ts index db79b2fe0e..6c9a87af02 100644 --- a/packages/agentic/storybook-desktop/src/cli/commandRunner.test.ts +++ b/packages/agentic/storybook-desktop/src/cli/commandRunner.test.ts @@ -19,10 +19,11 @@ describe('shared command output', () => { expect(NodeDesktopCommandRunner).toBe(SharedRunner); }); - test('retains attributed logs and forwards stderr without verbose', async () => { + function collect(options: { verbose?: boolean } = {}) { const stdout: string[] = []; const stderr: string[] = []; const runner = new NodeDesktopCommandRunner({ + ...options, output: { write: (text) => { stdout.push(String(text)); @@ -36,21 +37,56 @@ describe('shared command output', () => { }, }, }); + return { runner, stdout: () => stdout.join(''), stderr: () => stderr.join('') }; + } + + function onlyRun() { + const runs = path.join(root, 'artifacts', 'desktop-commands'); + return path.join(runs, fs.readdirSync(runs)[0]); + } + + test('retains attributed logs and reports only the outcome of a passing command without verbose', async () => { + const { runner, stdout, stderr } = collect(); try { await runner.run(command("process.stdout.write('information\\n');process.stderr.write('diagnostic\\n');")); } finally { await runner.close(); } - expect(stderr.join('')).toContain('stderr: diagnostic'); - expect(stdout.join('')).not.toContain('stdout: information'); - const runs = path.join(root, 'artifacts', 'desktop-commands'); - const run = path.join(runs, fs.readdirSync(runs)[0]); + expect(stdout()).toContain('✔ fixture command'); + expect(stdout()).not.toContain('information'); + expect(stderr()).not.toContain('diagnostic'); + const run = onlyRun(); const log = path.join(run, 'commands', fs.readdirSync(path.join(run, 'commands'))[0]); expect(fs.readFileSync(log, 'utf8')).toContain('[stdout] information'); expect(fs.readFileSync(log, 'utf8')).toContain('[stderr] diagnostic'); expect(JSON.parse(fs.readFileSync(path.join(run, 'run.json'), 'utf8')).status).toBe('passed'); }); + test('streams command output live under the command name with verbose', async () => { + const { runner, stdout, stderr } = collect({ verbose: true }); + try { + await runner.run(command("process.stdout.write('information\\n');process.stderr.write('diagnostic\\n');")); + } finally { + await runner.close(); + } + expect(stdout()).toContain('[fixture command] stdout: information'); + expect(stderr()).toContain('[fixture command] stderr: diagnostic'); + }); + + test('shows the end of the log under the header of a failed command', async () => { + const { runner, stdout, stderr } = collect(); + try { + await expect(runner.run(command("process.stderr.write('diagnostic\\n');process.exitCode=7;"))).rejects.toThrow('exited with code 7'); + } finally { + await runner.close(); + } + expect(stderr()).toContain('✖ fixture command'); + expect(stderr()).toContain('exit code 7'); + expect(stderr()).toContain('[stderr] diagnostic'); + expect(stderr()).toContain('Full log:'); + expect(stdout()).not.toContain('diagnostic'); + }); + test('keeps the original command failure in the shared result', async () => { const runner = new NodeDesktopCommandRunner({ output: { write: () => true }, errorOutput: { write: () => true } }); try { diff --git a/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts b/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts index b1ac6880fd..88ae7e784a 100644 --- a/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts +++ b/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts @@ -57,7 +57,7 @@ export function createDesktopStorybookCommand(options: CreateDesktopStorybookCom .addOption(new Option('--reporter ', 'output format').choices([...desktopReporterModes])) .option('--json', 'alias for --reporter json') .addOption(new Option('--events ', 'NDJSON event detail').choices([...desktopEventLevels])) - .option('-v, --verbose', 'stream informational output as well as live errors and test progress'); + .option('-v, --verbose', 'stream command output live, with start markers and log paths, instead of only the outcome of each step'); let apiPromise: Promise | undefined; const getApi = (flags: PlatformFlags) => diff --git a/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts b/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts index c07b9ee7b2..d18d37bc8d 100644 --- a/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts +++ b/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts @@ -123,7 +123,7 @@ describe('runDesktopStorybookSmokeTests', () => { }); expect(deleteSession).toHaveBeenCalledTimes(1); expect(result.tests).toEqual(planResult().tests); - expect(formatDesktopStorybookSmokeTestSummary(result)).toBe('Ran 1 desktop story tests (1 passed, 0 skipped).'); + expect(formatDesktopStorybookSmokeTestSummary(result)).toBe('1 of 1 story tests executed: 1 passed, 0 failed, 0 skipped'); expect(JSON.parse(fs.readFileSync(path.join(root, 'artifacts/windows/desktop-driver/run.json'), 'utf8'))).toMatchObject({ status: 'passed', tests: [{ storyId: buttonId }], @@ -168,7 +168,7 @@ describe('runDesktopStorybookSmokeTests', () => { ); expect(result.tests.map(({ storyId }) => storyId)).toEqual([buttonId, inputId]); expect(result.wdio?.map(({ storyId }) => storyId)).toEqual([buttonId, inputId]); - expect(formatDesktopStorybookSmokeTestSummary(result)).toBe('Ran 4 desktop story tests (4 passed, 0 skipped).'); + expect(formatDesktopStorybookSmokeTestSummary(result)).toBe('4 of 4 story tests executed: 4 passed, 0 failed, 0 skipped'); }); test('applies focused smoke selectors to both plans and WDIO while preserving owned sessions', async () => { @@ -404,14 +404,84 @@ describe('runDesktopStorybookSmokeTests', () => { await expect(runDesktopStorybookSmokeTests(options, jest.fn(), runWdio)).rejects.toThrow('only skipped or unsupported'); }); - test('breaks unsupported down in the summary only when something is unsupported', () => { + test('counts what executed against what was selected, and says so when nothing executed', () => { const wdio = [{ storyId: buttonId, status: 'passed' as const, durationMs: 1 }]; - expect(formatDesktopStorybookSmokeTestSummary({ ...planResult(), wdio })).toBe('Ran 2 desktop story tests (2 passed, 0 skipped).'); + expect(formatDesktopStorybookSmokeTestSummary({ ...planResult(), wdio })).toBe( + '2 of 2 story tests executed: 2 passed, 0 failed, 0 skipped', + ); expect( formatDesktopStorybookSmokeTestSummary({ ...planResult(), wdio: [...wdio, { storyId: inputId, status: 'unsupported' as const, gap: 'unsupported' as const, durationMs: 0 }], }), - ).toBe('Ran 3 desktop story tests (2 passed, 0 skipped, 1 unsupported (1 declared, 0 quarantined)).'); + ).toBe('2 of 3 story tests executed: 2 passed, 0 failed, 0 skipped, 1 unsupported (1 declared, 0 quarantined)'); + expect( + formatDesktopStorybookSmokeTestSummary({ + ...planResult(), + tests: [], + wdio: [{ storyId: inputId, status: 'unsupported' as const, gap: 'quarantine' as const, durationMs: 0 }], + }), + ).toBe('no story tests executed (0 of 1): 0 passed, 0 failed, 0 skipped, 1 unsupported (0 declared, 1 quarantined)'); + }); + + describe('outcome header', () => { + const outcomes = (emit: jest.Mock) => + emit.mock.calls.map(([event]) => event).filter(({ kind, payload }) => kind === 'scope-end' && payload.summary !== undefined); + + test('summarizes a passing run with the count of tests that executed', async () => { + const { report, emit } = fakeReport(root); + options.report = report; + const connect: DesktopStorybookSmokeConnector = async () => ({ + delete: async () => undefined, + listStories: async () => manifest, + runStoryTests: async () => planResult(), + }); + await runDesktopStorybookSmokeTests(options, connect); + expect(outcomes(emit)).toEqual([ + { kind: 'scope-end', payload: { status: 'passed', summary: '1 of 1 story tests executed: 1 passed, 0 failed, 0 skipped' } }, + ]); + }); + + test('summarizes a failing run with the counts the failure left, not only the error', async () => { + const { report, emit } = fakeReport(root); + options.report = report; + options.manifest = { ...manifest, entries: [{ ...story(buttonId, true), tests: undefined }] }; + const runWdio = jest.fn(async () => { + throw new Error('callback failed'); + }); + await expect(runDesktopStorybookSmokeTests(options, jest.fn(), runWdio)).rejects.toThrow('callback failed'); + expect(outcomes(emit)).toEqual([ + { kind: 'scope-end', payload: { status: 'failed', summary: '1 of 1 story tests executed: 0 passed, 1 failed, 0 skipped' } }, + ]); + }); + + test('says how many stories were not run when a failure stops the run early', async () => { + const { report, emit } = fakeReport(root); + options.report = report; + options.manifest = { ...manifest, entries: [buttonId, inputId].map((id) => ({ ...story(id, true), tests: undefined })) }; + const runWdio = jest.fn(async () => { + throw new Error('callback failed'); + }); + await expect(runDesktopStorybookSmokeTests(options, jest.fn(), runWdio)).rejects.toThrow('callback failed'); + expect(outcomes(emit)).toEqual([ + { + kind: 'scope-end', + payload: { + status: 'failed', + summary: '1 of 1 story tests executed: 0 passed, 1 failed, 0 skipped (stopped early; 1 of 2 stories not run)', + }, + }, + ]); + }); + + test('names the error when the run fails before any test is counted', async () => { + const { report, emit } = fakeReport(root); + options.report = report; + options.manifest = { ...manifest, entries: [] }; + await expect(runDesktopStorybookSmokeTests(options, jest.fn())).rejects.toThrow('No desktop story tests'); + expect(outcomes(emit)).toEqual([ + { kind: 'scope-end', payload: { status: 'failed', summary: expect.stringContaining('No desktop story tests') } }, + ]); + }); }); }); diff --git a/packages/agentic/storybook-desktop/src/cli/smokeTests.ts b/packages/agentic/storybook-desktop/src/cli/smokeTests.ts index c45a586c70..245b4f603f 100644 --- a/packages/agentic/storybook-desktop/src/cli/smokeTests.ts +++ b/packages/agentic/storybook-desktop/src/cli/smokeTests.ts @@ -6,7 +6,7 @@ import type { DesktopStoryRunResult, DesktopStoryTestResult } from '@fluentui-re import { ArtifactManager } from '@fluentui-react-native/desktop-driver/artifacts'; import { selectDesktopStoryTests } from '@fluentui-react-native/desktop-driver/runner'; import { connectDesktopWebdriver } from '@fluentui-react-native/desktop-driver/wdio'; -import type { DesktopReportWriter } from '@fluentui-react-native/desktop-test/reporting'; +import { runDesktopOperation, type DesktopOperation, type DesktopReportWriter } from '@fluentui-react-native/desktop-test/reporting'; import type { DesktopWebdriverOptions, DesktopWebdriverSession } from '@fluentui-react-native/desktop-driver/wdio'; import type { DesktopStorybookConfig } from '../config/makeDesktopStorybookConfig.js'; @@ -44,10 +44,23 @@ export type DesktopStorybookSmokeTestResult = DesktopStoryRunResult & { wdio?: readonly WdioStoryTestResult[]; }; -export async function runDesktopStorybookSmokeTests( +/** + * Runs the story tests as one reported operation, so the run ends with a single header that says how many tests ran + * and how many of them passed, whether the run passed or failed. + */ +export function runDesktopStorybookSmokeTests( options: DesktopStorybookSmokeTestOptions, connect: DesktopStorybookSmokeConnector = connectDesktopWebdriver, runWdio: typeof runWdioStoryTests = runWdioStoryTests, +): Promise { + return runDesktopOperation(options.report, { label: 'Story tests' }, (operation) => runSmokeTests(options, connect, runWdio, operation)); +} + +async function runSmokeTests( + options: DesktopStorybookSmokeTestOptions, + connect: DesktopStorybookSmokeConnector, + runWdio: typeof runWdioStoryTests, + operation: DesktopOperation, ): Promise { const { config, manifest, platform } = options; if (manifest.endpoint !== platform) { @@ -90,7 +103,9 @@ export async function runDesktopStorybookSmokeTests( // Every selected case is announced up front, so a fail-fast stop leaves the remaining cases accounted for as not run. for (const payload of plannedWdioWork(wdioSelection, wdioSettings)) options.report?.emit({ kind: 'work-selected', payload }); + let attempted = 0; for (const storyId of storyIds) { + attempted++; const scope = `smoke:${storyId}`; options.report?.emit( { kind: 'scope-start', payload: { label: storyId, category: 'phase' } }, @@ -193,6 +208,10 @@ export async function runDesktopStorybookSmokeTests( tests, wdio, }; + const notReached = storyIds.length - attempted; + operation.summarize( + `${formatDesktopStorybookSmokeTestSummary(result)}${notReached > 0 ? ` (stopped early; ${notReached} of ${storyIds.length} stories not run)` : ''}`, + ); try { const reportPath = artifacts.writeMetadata('run', { ...result, @@ -310,9 +329,17 @@ async function runStoryPlans( return result; } -export function formatDesktopStorybookSmokeTestSummary(result: DesktopStorybookSmokeTestResult): string { +/** + * The count that tells a reader whether tests actually ran: how many of the selected story tests executed, and how they + * ended. A skipped or unsupported test is selected but never runs, so a run of only those reads as `no story tests executed`. + */ +export function formatDesktopStorybookSmokeTestSummary(result: Pick): string { const results = [...result.tests, ...(result.wdio ?? [])]; - return `Ran ${results.length} desktop story tests (${describeWdioCounts(results)}).`; + const executed = results.filter(({ status }) => status !== 'skipped' && status !== 'unsupported').length; + const counts = describeWdioCounts(results); + return executed === 0 + ? `no story tests executed (0 of ${results.length}): ${counts}` + : `${executed} of ${results.length} story tests executed: ${counts}`; } function errorMessage(error: unknown): string { diff --git a/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts b/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts index f118e9e801..20788fd30c 100644 --- a/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts +++ b/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts @@ -327,9 +327,14 @@ function createIntegrationManifest(temporaryDirectory: string, storybookPort: nu describe('PowerShell owned service integration', () => { const windowsTest = process.platform === 'win32' ? test : test.skip; - windowsTest.each([false, true])( - 'forwards live grandchild diagnostics and drains services (failure=%s)', - async (failService) => { + windowsTest.each([ + { verbose: true, failService: false }, + { verbose: true, failService: true }, + { verbose: false, failService: false }, + { verbose: false, failService: true }, + ])( + 'streams grandchild diagnostics live only with verbose, keeps them in the log, and drains services (%j)', + async ({ verbose, failService }) => { const port = await getAvailablePort(); const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-powershell-services-')); const output: string[] = []; @@ -345,6 +350,7 @@ describe('PowerShell owned service integration', () => { output: sink, errorOutput: sink, summary: false, + verbose, }); const transport = await startReportTransport(reporter); const runner = new NodeDesktopCommandRunner({ reporter, environment: transport.environment }); @@ -384,17 +390,19 @@ describe('PowerShell owned service integration', () => { completed = true; }); try { - await deadline( - (async () => { - while (!output.join('').includes('live PowerShell grandchild diagnostic')) { - if (completed) throw new Error(output.join('')); - await new Promise((resolve) => setTimeout(resolve, 50)); - } - })(), - 15_000, - 'PowerShell diagnostic', - ); - expect(completed).toBe(false); + if (verbose) { + await deadline( + (async () => { + while (!output.join('').includes('live PowerShell grandchild diagnostic')) { + if (completed) throw new Error(output.join('')); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + })(), + 15_000, + 'PowerShell diagnostic', + ); + expect(completed).toBe(false); + } expect(await running.completed).toBe(failService ? 1 : 0); } finally { try { @@ -403,7 +411,10 @@ describe('PowerShell owned service integration', () => { await transport.close(); const result = await reporter.finish(); expect(result.status).toBe(failService ? 'failed' : 'passed'); - if (failService) expect(output.join('')).toContain('original nested service failure'); + const text = output.join(''); + if (failService) expect(text).toContain('original nested service failure'); + // Without verbose the diagnostic stays in the log, so only a failed command shows it. + expect(text.includes('live PowerShell grandchild diagnostic')).toBe(verbose || failService); const pid: number = JSON.parse(fs.readFileSync(marker, 'utf8')); expect(() => process.kill(pid, 0)).toThrow(); } finally { diff --git a/packages/agentic/storybook-desktop/src/testing/laneVerdict.test.ts b/packages/agentic/storybook-desktop/src/testing/laneVerdict.test.ts index 845858dd7f..a4ade100cf 100644 --- a/packages/agentic/storybook-desktop/src/testing/laneVerdict.test.ts +++ b/packages/agentic/storybook-desktop/src/testing/laneVerdict.test.ts @@ -62,7 +62,7 @@ describe('WDIO cohort verdict', () => { }); test('summarizes counts and breaks unsupported down only when present', () => { - expect(describeWdioCounts([result('a', 'passed'), result('b', 'skipped')])).toBe('1 passed, 1 skipped'); + expect(describeWdioCounts([result('a', 'passed'), result('b', 'skipped')])).toBe('1 passed, 0 failed, 1 skipped'); expect( describeWdioCounts([ result('a', 'passed'), @@ -70,6 +70,18 @@ describe('WDIO cohort verdict', () => { result('c', 'unsupported', 'quarantine'), result('d', 'unsupported'), ]), - ).toBe('1 passed, 0 skipped, 3 unsupported (1 declared, 1 quarantined)'); + ).toBe('1 passed, 0 failed, 0 skipped, 3 unsupported (1 declared, 1 quarantined)'); + }); + + test('counts every status that is not a pass, a skip or unsupported as failed', () => { + expect( + describeWdioCounts([ + { status: 'passed' }, + { status: 'failed' }, + { status: 'timed-out' }, + { status: 'cancelled' }, + { status: 'infrastructure-error' }, + ]), + ).toBe('1 passed, 4 failed, 0 skipped'); }); }); diff --git a/packages/agentic/storybook-desktop/src/testing/laneVerdict.ts b/packages/agentic/storybook-desktop/src/testing/laneVerdict.ts index e33f24a8fc..942461a3f8 100644 --- a/packages/agentic/storybook-desktop/src/testing/laneVerdict.ts +++ b/packages/agentic/storybook-desktop/src/testing/laneVerdict.ts @@ -20,10 +20,16 @@ export function hasExecutedWdioCase(results: readonly Pick status === 'passed' || status === 'failed'); } -/** Counts for a summary line. The unsupported breakdown appears only when something is unsupported, so other summaries are unchanged. */ -export function describeWdioCounts(results: readonly { status: string; gap?: WdioCaseOutcome['gap'] }[]): string { +type CountedResult = { status: string; gap?: WdioCaseOutcome['gap'] }; + +// Static plans report several kinds of failure, such as `timed-out`, so a failure is anything that is not one of these. +const notFailures = new Set(['passed', 'skipped', 'unsupported']); + +/** Counts for a summary line. The unsupported breakdown appears only when something is unsupported. */ +export function describeWdioCounts(results: readonly CountedResult[]): string { const count = (status: string) => results.filter((result) => result.status === status).length; - const parts = [`${count('passed')} passed`, `${count('skipped')} skipped`]; + const failed = results.filter(({ status }) => !notFailures.has(status)).length; + const parts = [`${count('passed')} passed`, `${failed} failed`, `${count('skipped')} skipped`]; const unsupported = results.filter(({ status }) => status === 'unsupported'); if (unsupported.length) { parts.push( diff --git a/yarn.lock b/yarn.lock index 78d0c26639..15e6bec892 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3340,7 +3340,6 @@ __metadata: dependencies: "@fluentui-react-native/desktop-driver": "workspace:*" "@fluentui-react-native/scripts": "workspace:*" - "@rnx-kit/reporter": "catalog:" "@types/cross-spawn": "npm:6.0.6" "@wdio/cli": "catalog:" "@wdio/jasmine-framework": "catalog:" From 1424619fcf9825c9a088939cb90cb95c1ca96539 Mon Sep 17 00:00:00 2001 From: Jason Morse Date: Mon, 9 Nov 2026 07:56:41 -0800 Subject: [PATCH 2/2] Pin storybook-desktop console tests to local reporter mode On GitHub Actions runners the reporter selects github-actions mode from GITHUB_ACTIONS, which groups passing command output, so tests asserting local quiet output failed in CI. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9305975d-a017-4007-beb2-68a4684cbbc4 --- .../src/cli/commandRunner.test.ts | 16 +++++++++++++++- .../src/driver/serverIntegration.test.ts | 5 +++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/agentic/storybook-desktop/src/cli/commandRunner.test.ts b/packages/agentic/storybook-desktop/src/cli/commandRunner.test.ts index 6c9a87af02..7f5839edf5 100644 --- a/packages/agentic/storybook-desktop/src/cli/commandRunner.test.ts +++ b/packages/agentic/storybook-desktop/src/cli/commandRunner.test.ts @@ -5,11 +5,25 @@ import { NodeDesktopCommandRunner as SharedRunner } from '@fluentui-react-native import { NodeDesktopCommandRunner, type PreparedDesktopCommand } from './commandRunner.js'; describe('shared command output', () => { + // These cases check local console output; on a GitHub Actions runner the owned reporter would otherwise group it. + const consoleEnvironment = ['GITHUB_ACTIONS', 'DESKTOP_REPORTER'] as const; + let savedEnvironment: Partial>; let root: string; beforeEach(() => { + savedEnvironment = {}; + for (const name of consoleEnvironment) { + savedEnvironment[name] = process.env[name]; + delete process.env[name]; + } root = fs.mkdtempSync(path.join(os.tmpdir(), 'storybook-command-output-')); }); - afterEach(() => fs.rmSync(root, { recursive: true, force: true })); + afterEach(() => { + for (const name of consoleEnvironment) { + if (savedEnvironment[name] === undefined) delete process.env[name]; + else process.env[name] = savedEnvironment[name]; + } + fs.rmSync(root, { recursive: true, force: true }); + }); function command(source: string): PreparedDesktopCommand { return { command: process.execPath, args: ['-e', source], cwd: root, env: {}, label: 'fixture command' }; diff --git a/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts b/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts index 20788fd30c..142615b7d1 100644 --- a/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts +++ b/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts @@ -31,6 +31,7 @@ describe('desktop Storybook server integration', () => { const reporter = new DesktopRunReporter({ artifactsRoot: temporaryDirectory, kind: 'lifecycle', + reporterMode: 'plain', output: sink, errorOutput: sink, summary: false, @@ -116,6 +117,7 @@ describe('desktop Storybook server integration', () => { const reporter = new DesktopRunReporter({ artifactsRoot: temporaryDirectory, kind: 'lifecycle', + reporterMode: 'plain', output: sink, errorOutput: sink, summary: false, @@ -168,6 +170,7 @@ describe('desktop Storybook server integration', () => { const reporter = new DesktopRunReporter({ artifactsRoot: temporaryDirectory, kind: 'lifecycle', + reporterMode: 'plain', output: sink, errorOutput: sink, summary: false, @@ -236,6 +239,7 @@ describe('desktop Storybook server integration', () => { const reporter = new DesktopRunReporter({ artifactsRoot: temporaryDirectory, kind: 'lifecycle', + reporterMode: 'plain', output: sink, errorOutput: sink, summary: false, @@ -347,6 +351,7 @@ describe('PowerShell owned service integration', () => { const reporter = new DesktopRunReporter({ artifactsRoot: temporaryDirectory, kind: 'lifecycle', + reporterMode: 'plain', output: sink, errorOutput: sink, summary: false,