diff --git a/.changeset/desktop-persisted-run-queries.md b/.changeset/desktop-persisted-run-queries.md new file mode 100644 index 0000000000..58ee4ad73a --- /dev/null +++ b/.changeset/desktop-persisted-run-queries.md @@ -0,0 +1,5 @@ +--- +'@fluentui-react-native/desktop-test': patch +--- + +Add authorized bounded persisted run/status/artifact queries with opaque IDs, expiring read grants, explicit omissions, and fail-closed terminal evidence. Persist active collector snapshots without starting or replaying workloads when readers reconnect. diff --git a/packages/agentic/desktop-test/README.md b/packages/agentic/desktop-test/README.md index d6bd4db16f..fda75cc0e0 100644 --- a/packages/agentic/desktop-test/README.md +++ b/packages/agentic/desktop-test/README.md @@ -2,7 +2,10 @@ `@fluentui-react-native/desktop-test` provides application-neutral configuration, native service composition, process lifetimes, standalone WDIO/Jasmine suites, -and reporting for macOS, Windows, and Win32. It uses +and reporting for macOS, Windows, and Win32. Authorized, bounded persisted +run/status/artifact reads are available from the direct +`./reporting/run-queries` entry; see [reporting](references/reporting.md). +It uses `@fluentui-react-native/desktop-driver`; Storybook support is an optional integration in `@fluentui-react-native/storybook-desktop`. diff --git a/packages/agentic/desktop-test/package.json b/packages/agentic/desktop-test/package.json index 910b5c8418..f86c5cecd6 100644 --- a/packages/agentic/desktop-test/package.json +++ b/packages/agentic/desktop-test/package.json @@ -40,6 +40,11 @@ "import": "./lib/reporting/index.js", "default": "./src/reporting/index.ts" }, + "./reporting/run-queries": { + "types": "./lib/reporting/runQueries.d.ts", + "import": "./lib/reporting/runQueries.js", + "default": "./src/reporting/runQueries.ts" + }, "./commands": { "types": "./lib/commands.d.ts", "import": "./lib/commands.js", diff --git a/packages/agentic/desktop-test/references/current-status.md b/packages/agentic/desktop-test/references/current-status.md index ce464c77fa..0e6bd49ec4 100644 --- a/packages/agentic/desktop-test/references/current-status.md +++ b/packages/agentic/desktop-test/references/current-status.md @@ -32,7 +32,8 @@ and its sub-issues. | MCP/WDIO handoff | Work in progress | Must await drain/deletion; attached-app handoff is independent of future supervisor-owned launch. | | Supervisor-owned app lifetime | Work in progress | Current session-owned launch/externally owned attach semantics remain; never silently retain or terminate another owner's app. | | Cross-service/multi-target claims | Work in progress | Target aliases and input mutexes are not exact-application exclusion; resolve registered-ID collisions before expanding hosts. | -| Bounded persisted run-query/control API | Work in progress | Schemas belong here. Agents may eventually select only declared workload IDs, not commands/paths/environment. | +| Bounded persisted run/status/artifact queries | Done | Harness-local read service/client; atomic snapshots, terminal evidence, scoped expiring grants, bounded pages and allowlisted opaque artifact IDs. No management transport. | +| Workload management/control API | Work in progress | Future management selects only declared workload IDs, not commands/paths/environment; lifecycle/handoff gates still apply. | | Automatic service reuse and per-user daemon | Deferred | Requires exact identity/config matching and evidence of need. Never adopt an unknown port occupant. | See [lifecycle](lifecycle.md), [commands](commands.md), and diff --git a/packages/agentic/desktop-test/references/reporting.md b/packages/agentic/desktop-test/references/reporting.md index 6d9723b746..164b05dc64 100644 --- a/packages/agentic/desktop-test/references/reporting.md +++ b/packages/agentic/desktop-test/references/reporting.md @@ -64,3 +64,85 @@ directory or render a duplicate summary. Story-specific readiness, authenticated channel events, and callback execution are owned by `@fluentui-react-native/storybook-desktop`. + +## Persisted read queries + +The direct `@fluentui-react-native/desktop-test/reporting/run-queries` entry +exports `DesktopRunQueryService`, `DesktopRunQueryClient`, and their request and +response contracts. This is a harness-local read API, not a management listener, +MCP tool, or execution/recovery API. Its credentials are distinct from driver +control grants. Connecting a reader only binds a credential; it never creates +a workload, producer, controller session, or application. + +The collector atomically replaces `run.json` with an active snapshot after each +batch of events (coalesced within a JavaScript turn). Active snapshots have +`terminal: false` and cannot have a passing status. Only successful final +report persistence after producer/output accounting sets `terminal: true`. +Terminal means collection finished, not that coverage passed. A crashed +collector leaves its last snapshot nonterminal. Legacy reports lacking the +marker remain incomplete in queries, even if they say `passed`. Existing +`finishedAt` is the snapshot observation time until the terminal marker is set. +Persisted tests, work, failures, retries, and skips remain the authoritative +source; page size and worker exit codes do not decide coverage. + +Trusted host code registers exact collector `{ runId, root, expiresAt }` +records and an optional `textArtifacts` relative-path allowlist, then issues a +read grant with exact `runIds`, an expiry, and optional `artifacts` permission: + +```ts +import { DesktopRunQueryService } from '@fluentui-react-native/desktop-test/reporting/run-queries'; + +const queries = new DesktopRunQueryService(registeredSecrets); +queries.registerRun({ + runId: reporter.runId, + root: reporter.root, + expiresAt: retentionDeadline, + textArtifacts: ['commands/build.log'], +}); +const credential = queries.issueGrant({ + runIds: [reporter.runId], + expiresAt: readerDeadline, + artifacts: true, +}); +const client = queries.connect(credential); +const status = await client.query({ operation: 'status', runId: reporter.runId }); +``` + +Registration and grant issuance are **trusted operator/host operations**, never +model-facing inputs. Credentials stay in the host/client binding; requests and +persisted results contain no credentials. Host restart requires explicit +re-registration of retained runs and fresh grants, never execution replay. +The host must register all relevant secrets (including credentials for any +external integration); `addSecret` can update the read redactor. The collector +redacts before persistence, and queries redact again before clipping text. + +Readers submit only an opaque run ID, a declared operation, and its bounded +parameters. `status` returns complete counts and terminal/coverage state. +`tests`, `work`, `failures`, and `artifacts` return pages with a default limit of +10 and maximum of 25, capped at 16 KiB. Page IDs do not expose producer IDs or +paths. Text fields are capped at 1 KiB after redaction. `omissions.items` counts +all items absent from that page (including preceding pages); +`omissions.textBytes` counts redacted UTF-8 bytes clipped within returned items. +Opaque `nextCursor` values are bound to the grant, run, collection, and exact +snapshot; a changed snapshot requires a new first page. The last 256 cursors +per grant are retained; older, forged, or cross-grant cursors fail explicitly. + +Artifact listing and content both require artifact permission. Listing exposes +only allowlisted artifacts recorded by the collector, with opaque artifact +IDs and descriptions, never paths. `artifact` accepts that ID, not a path, +URL, filename, or command. Content reads return at most 4 KiB of UTF-8 text +with explicit omitted byte counts; binary/invalid UTF-8 is rejected. Raw +`run.json` must be queried structurally, not allowlisted as a text artifact. +Allowlist entries use canonical forward-slash relative paths on every OS; +aliases such as `./run.json` cannot bypass the structural-only rule. +Inputs over 64 MiB for canonical results or 1 MiB for a text artifact fail +explicitly rather than parsing a truncated result as passing. Symlinks, hardlinks, +replaced roots, missing files, malformed results, unauthorized scope, and +expired credentials or retention all fail closed. + +`revokeGrant` affects already connected clients. `forgetRun` immediately removes +query authority, including old grants if the host later re-registers that ID. +Retention expiry is checked on every read before any +projection, even if the files remain on disk. This API never deletes collector +files; the host/operator owns filesystem retention cleanup. No reader can +extend retention, enumerate roots, create a grant, or restart a run. diff --git a/packages/agentic/desktop-test/src/reporting/collector.ts b/packages/agentic/desktop-test/src/reporting/collector.ts index aebbf72d3c..4acbd00b8d 100644 --- a/packages/agentic/desktop-test/src/reporting/collector.ts +++ b/packages/agentic/desktop-test/src/reporting/collector.ts @@ -87,6 +87,7 @@ export class DesktopRunReporter { private emittedProblemEvents = 0; private structuredOutputError?: Error; private terminal?: DesktopRunResult; + private snapshotQueued = false; constructor(options: DesktopRunReporterOptions) { this.options = options; @@ -127,6 +128,7 @@ export class DesktopRunReporter { this.artifacts.set('events.jsonl', 'Versioned event journal'); this.artifacts.set('run.json', 'Canonical run result'); if (options.junit !== false) this.artifacts.set('junit.xml', 'JUnit result'); + this.persistSnapshot(this.result()); if (this.reporterMode === 'ndjson') { const start = { schemaVersion: 1, @@ -255,6 +257,20 @@ export class DesktopRunReporter { } this.reduce(event); this.present(event); + if (!this.snapshotQueued) { + this.snapshotQueued = true; + queueMicrotask(() => { + this.snapshotQueued = false; + if (this.finished) return; + try { + this.persistSnapshot(this.result()); + } catch (error) { + const failure = this.failure(error, 'reporting'); + this.failures.set(failure.failureId, failure); + this.logger.error(`Cannot persist desktop run snapshot: ${failure.error.message}`); + } + }); + } } private reduce(event: CollectedDesktopEvent): void { @@ -497,7 +513,18 @@ export class DesktopRunReporter { fs.renameSync(temporary, this.artifactPath(name)); }; if (this.options.junit !== false) write('junit.xml', renderJUnit(result)); - write('run.json', JSON.stringify(result, null, 2) + '\n'); + this.persistSnapshot(result); + } + + private persistSnapshot(result: DesktopRunResult): void { + const temporary = this.artifactPath('run.json.tmp'); + const snapshot = this.sanitize({ + ...result, + terminal: result.terminal === true, + status: result.terminal === true || result.status === 'failed' ? result.status : 'incomplete', + }); + fs.writeFileSync(temporary, JSON.stringify(snapshot) + '\n', { mode: 0o600 }); + fs.renameSync(temporary, this.artifactPath('run.json')); } private async finalize(): Promise { @@ -603,6 +630,8 @@ export class DesktopRunReporter { // A closed stderr cannot carry the reporting diagnostic. } } + result = { ...this.result(), terminal: true }; + this.persist(result); this.finished = true; this.terminal = result; return result; diff --git a/packages/agentic/desktop-test/src/reporting/results.ts b/packages/agentic/desktop-test/src/reporting/results.ts index da08978f6e..2470c157bc 100644 --- a/packages/agentic/desktop-test/src/reporting/results.ts +++ b/packages/agentic/desktop-test/src/reporting/results.ts @@ -28,6 +28,8 @@ export type DesktopRunResult = { runId: string; kind: 'tests' | 'lifecycle'; status: TestStatus; + /** Only true after collector finalization; absent in legacy reports. */ + terminal?: boolean; startedAt: string; finishedAt: string; durationMs: number; diff --git a/packages/agentic/desktop-test/src/reporting/runQueries.test.ts b/packages/agentic/desktop-test/src/reporting/runQueries.test.ts new file mode 100644 index 0000000000..75a352a566 --- /dev/null +++ b/packages/agentic/desktop-test/src/reporting/runQueries.test.ts @@ -0,0 +1,368 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; + +import { DesktopRunReporter } from './collector.js'; +import { DesktopRunQueryService, validateDesktopRunQuery } from './runQueries.js'; +import type { DesktopRunPage, DesktopRunQueryClient } from './runQueries.js'; +import type { DesktopRunResult } from './results.js'; + +describe('persisted desktop run queries', () => { + let root: string; + let reporter: DesktopRunReporter; + let service: DesktopRunQueryService; + let token: string; + let client: DesktopRunQueryClient; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-run-queries-')); + reporter = new DesktopRunReporter({ + artifactsRoot: root, + output: { write: () => true }, + reporterMode: 'json', + summary: false, + secrets: ['collector-secret'], + }); + service = new DesktopRunQueryService(['reader-secret']); + service.registerRun({ + runId: reporter.runId, + root: reporter.root, + expiresAt: Date.now() + 60_000, + textArtifacts: ['commands/output.log'], + }); + token = service.issueGrant({ runIds: [reporter.runId], expiresAt: Date.now() + 60_000, artifacts: true }); + client = service.connect(token); + }); + + afterEach(async () => { + await reporter.finish(); + fs.rmSync(root, { force: true, recursive: true }); + }); + + function testResult(status: 'passed' | 'skipped' | 'failed', testId: string = randomUUID(), title = testId): void { + reporter.emit({ + kind: 'test-result', + payload: { + testId, + title, + status, + attempt: 0, + durationMs: 1, + ...(status === 'skipped' ? { reason: 'Unsupported endpoint' } : {}), + ...(status === 'failed' ? { failure: reporter.failure(new Error('Assertion failed'), 'test') } : {}), + }, + }); + } + + function rewrite(change: (result: DesktopRunResult) => void): void { + const file = path.join(reporter.root, 'run.json'); + const result = JSON.parse(fs.readFileSync(file, 'utf8')) as DesktopRunResult; + change(result); + fs.writeFileSync(file, JSON.stringify(result)); + } + + test('active snapshots are incomplete even after a passing test; reconnect never replays execution', async () => { + expect(await client.query({ operation: 'status', runId: reporter.runId })).toMatchObject({ + terminal: false, + status: 'incomplete', + counts: { selected: 0 }, + }); + testResult('passed'); + await reporter.flush(); + expect(JSON.parse(fs.readFileSync(path.join(reporter.root, 'run.json'), 'utf8'))).toMatchObject({ + terminal: false, + status: 'incomplete', + }); + const before = fs.readFileSync(path.join(reporter.root, 'events.jsonl'), 'utf8'); + const next = service.connect(token); + expect(await next.query({ operation: 'status', runId: reporter.runId })).toMatchObject({ + terminal: false, + status: 'incomplete', + counts: { passed: 1 }, + }); + expect(fs.readFileSync(path.join(reporter.root, 'events.jsonl'), 'utf8')).toBe(before); + await reporter.finish(); + expect(await next.query({ operation: 'status', runId: reporter.runId })).toMatchObject({ terminal: true, status: 'passed' }); + }); + + test('reader recreation over retained storage needs no collector or execution callback', async () => { + testResult('passed'); + await reporter.finish(); + const replacement = new DesktopRunQueryService(); + replacement.registerRun({ runId: reporter.runId, root: reporter.root, expiresAt: Date.now() + 60_000 }); + const credential = replacement.issueGrant({ runIds: [reporter.runId], expiresAt: Date.now() + 60_000 }); + expect(await replacement.connect(credential).query({ operation: 'status', runId: reporter.runId })).toMatchObject({ + terminal: true, + status: 'passed', + }); + expect(() => replacement.connect(token)).toThrow('unauthorized'); + }); + + test.each(['skipped', 'failed'] as const)('terminal %s work never projects success', async (status) => { + testResult(status); + await reporter.finish(); + expect(await client.query({ operation: 'status', runId: reporter.runId })).toMatchObject({ + terminal: true, + status: status === 'skipped' ? 'incomplete' : 'failed', + }); + }); + + test('mixed pass/skip retains coverage semantics', async () => { + testResult('passed'); + testResult('skipped'); + await reporter.finish(); + expect(await client.query({ operation: 'status', runId: reporter.runId })).toMatchObject({ + terminal: true, + status: 'passed', + counts: { passed: 1, skipped: 1 }, + }); + }); + + test('unfinished producers and work remain incomplete after finalization', async () => { + reporter.source('unfinished-worker'); + reporter.emit({ kind: 'work-selected', payload: { workId: 'unfinished', label: 'Selected work' } }); + testResult('passed'); + await reporter.finish(); + expect(await client.query({ operation: 'status', runId: reporter.runId })).toMatchObject({ terminal: true, status: 'incomplete' }); + expect(await client.query({ operation: 'work', runId: reporter.runId })).toMatchObject({ + items: [{ title: 'Selected work', status: 'not-run' }], + omissions: { items: 0 }, + }); + }); + + test('legacy reports without terminal evidence fail closed, regardless of claimed counts', async () => { + testResult('passed'); + await reporter.finish(); + rewrite((result) => { + delete result.terminal; + result.counts.passed = 9999; + }); + expect(await client.query({ operation: 'status', runId: reporter.runId })).toMatchObject({ + terminal: false, + status: 'incomplete', + counts: { passed: 1 }, + }); + }); + + test('large pages cannot hide a late failure and expose exact omission counts within the byte cap', async () => { + for (let index = 0; index < 200; index++) testResult('passed', `test-${index}`, 'wide title '.repeat(400)); + testResult('failed', 'late-failure'); + await reporter.finish(); + const page = (await client.query({ operation: 'tests', runId: reporter.runId, limit: 25 })) as DesktopRunPage; + expect(page.status).toBe('failed'); + expect(page.total).toBe(201); + expect(page.omissions.items).toBe(201 - page.items.length); + expect(page.omissions.textBytes).toBeGreaterThan(0); + expect(Buffer.byteLength(JSON.stringify(page))).toBeLessThanOrEqual(16 * 1024); + const seen = new Set(page.items.map(({ id }) => id)); + let cursor = page.nextCursor; + while (cursor) { + const next = (await client.query({ operation: 'tests', runId: reporter.runId, cursor, limit: 25 })) as DesktopRunPage; + for (const { id } of next.items) { + expect(seen.has(id)).toBe(false); + seen.add(id); + } + cursor = next.nextCursor; + } + expect(seen.size).toBe(201); + }); + + test('cursors are bound to grant, operation, run, and immutable snapshot', async () => { + testResult('passed'); + testResult('passed'); + await reporter.flush(); + const page = (await client.query({ operation: 'tests', runId: reporter.runId, limit: 1 })) as DesktopRunPage; + expect(page.nextCursor).toBeDefined(); + await expect(client.query({ operation: 'work', runId: reporter.runId, cursor: page.nextCursor })).rejects.toThrow('cursor'); + const other = service.connect(service.issueGrant({ runIds: [reporter.runId], expiresAt: Date.now() + 60_000 })); + await expect(other.query({ operation: 'tests', runId: reporter.runId, cursor: page.nextCursor })).rejects.toThrow('cursor'); + testResult('passed'); + await reporter.flush(); + await expect(client.query({ operation: 'tests', runId: reporter.runId, cursor: page.nextCursor })).rejects.toThrow('stale'); + }); + + test('revocation and unregistered run scope fail closed for already connected readers', async () => { + await expect(client.query({ operation: 'status', runId: randomUUID() })).rejects.toThrow('unauthorized'); + service.revokeGrant(token); + await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toThrow('unauthorized'); + expect(() => service.connect(token)).toThrow('unauthorized'); + }); + + test('a valid credential cannot read a different registered run', async () => { + const otherId = randomUUID(); + service.registerRun({ runId: otherId, root: reporter.root, expiresAt: Date.now() + 60_000 }); + await expect(client.query({ operation: 'status', runId: otherId })).rejects.toThrow('unauthorized'); + }); + + test('forgetting and re-registering a run cannot restore an old grant', async () => { + service.forgetRun(reporter.runId); + service.registerRun({ runId: reporter.runId, root: reporter.root, expiresAt: Date.now() + 60_000 }); + await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toThrow('unauthorized'); + }); + + test.each(['run.json', './run.json', 'commands/../run.json', '../outside.log'])( + 'trusted artifact registration rejects unsafe or raw canonical projection %s', + (artifact) => { + expect(() => + service.registerRun({ + runId: randomUUID(), + root: reporter.root, + expiresAt: Date.now() + 60_000, + textArtifacts: [artifact], + }), + ).toThrow(); + }, + ); + + test.each(['grant', 'retention', 'forget'] as const)('%s invalidation applies before projection', async (kind) => { + const scoped = new DesktopRunQueryService(); + scoped.registerRun({ + runId: reporter.runId, + root: reporter.root, + expiresAt: Date.now() + (kind === 'retention' ? 1000 : 60_000), + }); + const reader = scoped.connect( + scoped.issueGrant({ runIds: [reporter.runId], expiresAt: Date.now() + (kind === 'grant' ? 1000 : 60_000) }), + ); + if (kind === 'forget') scoped.forgetRun(reporter.runId); + const now = jest.spyOn(Date, 'now').mockReturnValue(Date.now() + 2000); + try { + await expect(reader.query({ operation: 'status', runId: reporter.runId })).rejects.toThrow('unauthorized'); + } finally { + now.mockRestore(); + } + }); + + test('only authorized opaque artifact IDs work; text is redacted before clipping and counted in UTF-8 bytes', async () => { + fs.mkdirSync(path.join(reporter.root, 'commands')); + const text = 'reader-secret '.repeat(500) + '😀'.repeat(500); + fs.writeFileSync(path.join(reporter.root, 'commands/output.log'), text); + reporter.emit({ kind: 'artifact', payload: { path: 'commands/output.log', description: 'Command output reader-secret' } }); + await reporter.flush(); + const page = (await client.query({ operation: 'artifacts', runId: reporter.runId })) as DesktopRunPage; + expect(JSON.stringify(page)).not.toContain('reader-secret'); + expect(JSON.stringify(page)).not.toContain(reporter.root); + expect(JSON.stringify(page)).not.toContain('commands/output.log'); + const artifact = await client.query({ operation: 'artifact', runId: reporter.runId, artifactId: page.items[0].id }); + expect('text' in artifact).toBe(true); + if (!('text' in artifact)) throw new Error('Expected artifact response.'); + expect(artifact.text).not.toContain('reader-secret'); + expect(artifact.text).toContain('[REDACTED]'); + expect(Buffer.byteLength(artifact.text)).toBeLessThanOrEqual(4096); + expect(artifact.omissions.bytes + Buffer.byteLength(artifact.text)).toBe( + Buffer.byteLength(text.replaceAll('reader-secret', '[REDACTED]')), + ); + const restricted = service.connect(service.issueGrant({ runIds: [reporter.runId], expiresAt: Date.now() + 60_000 })); + await expect(restricted.query({ operation: 'artifacts', runId: reporter.runId })).rejects.toThrow('unauthorized'); + await expect(restricted.query({ operation: 'artifact', runId: reporter.runId, artifactId: page.items[0].id })).rejects.toThrow( + 'unauthorized', + ); + await expect(client.query({ operation: 'artifact', runId: reporter.runId, artifactId: randomUUID() })).rejects.toThrow('unauthorized'); + }); + + test('collector and read-time secrets never appear in pages, including late registered secrets', async () => { + testResult('passed', 'sensitive-id', 'collector-secret reader-secret late-secret'); + reporter.addSecret('late-secret'); + await reporter.finish(); + const page = (await client.query({ operation: 'tests', runId: reporter.runId })) as DesktopRunPage; + expect(page.items[0].title).toBe('[REDACTED] [REDACTED] [REDACTED]'); + expect(JSON.stringify(page)).not.toContain('sensitive-id'); + }); + + test('failures are projected without stacks, contexts, or paths', async () => { + reporter.fail(new Error('Failure reader-secret')); + await reporter.finish(); + const page = (await client.query({ operation: 'failures', runId: reporter.runId })) as DesktopRunPage; + expect(page.items[0]).toMatchObject({ title: 'infrastructure', message: 'Failure [REDACTED]' }); + expect(JSON.stringify(page)).not.toContain('stack'); + expect(JSON.stringify(page)).not.toContain(reporter.root); + }); + + (process.platform === 'win32' ? test.skip : test)( + 'symlinked retained results are rejected (file symlinks need Windows privilege)', + async () => { + testResult('passed'); + await reporter.finish(); + const file = path.join(reporter.root, 'run.json'); + const retained = path.join(root, 'retained.json'); + fs.renameSync(file, retained); + fs.symlinkSync(retained, file); + await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toThrow('symlink'); + fs.unlinkSync(file); + fs.renameSync(retained, file); + }, + ); + + test('hardlinked and oversized retained results are rejected instead of truncated into success', async () => { + testResult('passed'); + await reporter.finish(); + const file = path.join(reporter.root, 'run.json'); + const linked = path.join(root, 'linked.json'); + fs.linkSync(file, linked); + await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toThrow('private regular file'); + fs.unlinkSync(linked); + fs.truncateSync(file, 64 * 1024 * 1024 + 1); + await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toThrow('limit'); + }); + + test('artifact limits, invalid UTF-8, and missing content fail explicitly', async () => { + fs.mkdirSync(path.join(reporter.root, 'commands')); + reporter.emit({ kind: 'artifact', payload: { path: 'commands/output.log', description: 'Output' } }); + await reporter.flush(); + const page = (await client.query({ operation: 'artifacts', runId: reporter.runId })) as DesktopRunPage; + const query = { operation: 'artifact' as const, runId: reporter.runId, artifactId: page.items[0].id }; + await expect(client.query(query)).rejects.toThrow(); + const file = path.join(reporter.root, 'commands/output.log'); + fs.writeFileSync(file, Buffer.from([0xff])); + await expect(client.query(query)).rejects.toThrow('encoded data'); + fs.truncateSync(file, 1024 * 1024 + 1); + await expect(client.query(query)).rejects.toThrow('limit'); + }); + + test('malformed/inconsistent persisted data never returns a success-shaped fallback', async () => { + testResult('passed'); + await reporter.finish(); + rewrite((result) => { + result.tests[0].status = 'failed'; + }); + + await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toThrow('disagrees'); + fs.writeFileSync(path.join(reporter.root, 'run.json'), '{'); + await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toThrow(); + }); + + test('snapshot persistence failures remain reporting failures after the writer recovers', async () => { + const write = fs.writeFileSync; + const injected = jest.spyOn(fs, 'writeFileSync').mockImplementation((file, ...args) => { + if (String(file).endsWith('run.json.tmp')) throw new Error('Injected snapshot persistence failure.'); + return write(file, ...args); + }); + try { + testResult('passed'); + await reporter.flush(); + } finally { + injected.mockRestore(); + } + await reporter.finish(); + expect(await client.query({ operation: 'status', runId: reporter.runId })).toMatchObject({ + terminal: true, + status: 'failed', + }); + expect(await client.query({ operation: 'failures', runId: reporter.runId })).toMatchObject({ + items: [expect.objectContaining({ title: 'reporting', message: 'Injected snapshot persistence failure.' })], + }); + }); + + test.each([ + { operation: 'artifact', artifactId: '../run.json' }, + { operation: 'status', path: '/tmp/run.json' }, + { operation: 'tests', limit: 0 }, + { operation: 'tests', limit: 26 }, + { operation: 'tests', limit: 1.5 }, + { operation: 'tests', cursor: 'model supplied path' }, + { operation: 'restart' }, + ])('rejects invalid request %j', (input) => { + expect(() => validateDesktopRunQuery({ runId: reporter.runId, ...input })).toThrow(); + }); +}); diff --git a/packages/agentic/desktop-test/src/reporting/runQueries.ts b/packages/agentic/desktop-test/src/reporting/runQueries.ts new file mode 100644 index 0000000000..e75938f236 --- /dev/null +++ b/packages/agentic/desktop-test/src/reporting/runQueries.ts @@ -0,0 +1,423 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { createHash, randomUUID } from 'node:crypto'; +import { isNativeError } from 'node:util/types'; + +import { desktopArtifactPath } from './collector.js'; +import { DesktopReportedError, reportStatuses, redactDesktopSecrets } from './protocol.js'; +import type { TestStatus } from './protocol.js'; +import { summarizeDesktopRun } from './results.js'; +import type { DesktopRunResult } from './results.js'; + +export type DesktopRunQuery = + | { operation: 'status'; runId: string } + | { operation: 'tests' | 'work' | 'failures' | 'artifacts'; runId: string; limit?: number; cursor?: string } + | { operation: 'artifact'; runId: string; artifactId: string }; + +export type DesktopRunStatus = { + schemaVersion: 1; + runId: string; + terminal: boolean; + status: TestStatus; + counts: DesktopRunResult['counts']; +}; +export type DesktopRunQueryItem = { + id: string; + title: string; + status?: TestStatus; + message?: string; +}; +export type DesktopRunPage = { + schemaVersion: 1; + runId: string; + terminal: boolean; + status: TestStatus; + items: DesktopRunQueryItem[]; + total: number; + omissions: { items: number; textBytes: number }; + nextCursor?: string; +}; +export type DesktopArtifactQueryResult = { + schemaVersion: 1; + runId: string; + artifactId: string; + text: string; + omissions: { bytes: number }; +}; +export type DesktopRunQueryResponse = DesktopRunStatus | DesktopRunPage | DesktopArtifactQueryResult; + +export type DesktopRunRegistration = { + runId: string; + root: string; + expiresAt: number; + /** Only these trusted, relative text artifacts can be read; run.json is queried structurally instead. */ + textArtifacts?: readonly string[]; +}; +export type DesktopRunReadGrant = { + runIds: readonly string[]; + expiresAt: number; + artifacts?: boolean; +}; +type RegisteredRun = DesktopRunRegistration & { + device: number; + inode: number; + artifacts: Map; +}; +type Grant = DesktopRunReadGrant & { + cursors: Map; +}; + +const maxResultBytes = 64 * 1024 * 1024; +const maxArtifactBytes = 1024 * 1024; +const maxResponseBytes = 16 * 1024; +const maxTextBytes = 1024; +const maxCursors = 256; +const unavailable = () => new Error('Desktop run query is unavailable or unauthorized.'); + +/** + * Trusted host composition owns registrations and credentials. Readers cannot start work, discover roots, + * register artifacts, or gain permissions by reconnecting. This is not a management listener. + */ +export class DesktopRunQueryService { + private readonly runs = new Map(); + private readonly grants = new Map(); + private readonly secrets = new Set(); + + constructor(secrets: readonly string[] = []) { + for (const secret of secrets) this.addSecret(secret); + } + + addSecret(secret: string): void { + if (secret) this.secrets.add(secret); + } + + registerRun(registration: DesktopRunRegistration): void { + opaqueId(registration.runId); + deadline(registration.expiresAt); + if (this.runs.has(registration.runId)) throw new Error('Desktop run is already registered.'); + const root = fs.realpathSync(registration.root); + const stat = fs.statSync(root); + if (!stat.isDirectory()) throw new Error('Desktop run root must be a directory.'); + const artifacts = new Map(); + for (const relative of registration.textArtifacts ?? []) { + const resolved = desktopArtifactPath(root, relative); + if (relative.includes('\\') || path.relative(root, resolved).split(path.sep).join('/') !== relative || relative === 'run.json') + throw new Error('Text artifact paths must be canonical relative paths other than run.json.'); + artifacts.set(randomUUID(), relative); + } + this.runs.set(registration.runId, { ...registration, root, device: stat.dev, inode: stat.ino, artifacts }); + } + + forgetRun(runId: string): void { + this.runs.delete(runId); + for (const grant of this.grants.values()) { + grant.runIds = grant.runIds.filter((id) => id !== runId); + for (const [cursor, scope] of grant.cursors) if (scope.runId === runId) grant.cursors.delete(cursor); + } + } + + issueGrant(input: DesktopRunReadGrant): string { + deadline(input.expiresAt); + if (!input.runIds.length || input.runIds.some((id) => !this.runs.has(id))) throw unavailable(); + const token = randomUUID(); + this.addSecret(token); + this.grants.set(token, { ...input, runIds: [...input.runIds], cursors: new Map() }); + return token; + } + + revokeGrant(token: string): void { + this.grants.delete(token); + } + + connect(token: string): DesktopRunQueryClient { + if (!this.grants.has(token) || this.grants.get(token)!.expiresAt <= Date.now()) throw unavailable(); + return new DesktopRunQueryClient((query) => this.query(token, query)); + } + + query(token: string, input: unknown): DesktopRunQueryResponse { + try { + return this.read(token, input); + } catch (error) { + throw new Error(this.redact(isNativeError(error) ? error.message : 'Desktop run query failed.')); + } + } + + private read(token: string, input: unknown): DesktopRunQueryResponse { + validateDesktopRunQuery(input); + const grant = this.grants.get(token); + const run = this.runs.get(input.runId); + if (!grant || !run || !grant.runIds.includes(input.runId) || grant.expiresAt <= Date.now() || run.expiresAt <= Date.now()) + throw unavailable(); + const rootStat = fs.lstatSync(run.root); + if (!rootStat.isDirectory() || rootStat.dev !== run.device || rootStat.ino !== run.inode) throw unavailable(); + if ((input.operation === 'artifact' || input.operation === 'artifacts') && !grant.artifacts) throw unavailable(); + const retained = readRetainedText(run.root, 'run.json', maxResultBytes); + const digest = createHash('sha256').update(retained).digest('hex'); + const result = parseResult(retained, run.runId); + if (input.operation === 'artifact') { + const relative = run.artifacts.get(input.artifactId); + if (!relative || !result.artifacts.some((artifact) => artifact.path === relative)) throw unavailable(); + const text = this.redact(readRetainedText(run.root, relative, maxArtifactBytes)); + const clipped = clip(text, 4 * 1024); + return { + schemaVersion: 1, + runId: run.runId, + artifactId: input.artifactId, + text: clipped.text, + omissions: { bytes: clipped.omitted }, + }; + } + const terminal = result.terminal === true; + // Recompute coverage from complete persisted data, never from a truncated page or producer exit code. + const canonical = summarizeDesktopRun({ + ...result, + optional: result.status === 'skipped', + incomplete: !terminal || ['incomplete', 'cancelled', 'not-run', 'unsupported'].includes(result.status), + }); + if (result.status === 'failed') canonical.status = 'failed'; + if (input.operation === 'status') + return { schemaVersion: 1, runId: run.runId, terminal, status: canonical.status, counts: canonical.counts }; + let offset = 0; + if (input.cursor) { + const cursor = grant.cursors.get(input.cursor); + if (!cursor || cursor.runId !== run.runId || cursor.operation !== input.operation || cursor.digest !== digest) + throw new Error('Desktop run cursor is stale or unauthorized; request a new first page.'); + offset = cursor.offset; + } + const items = this.items(input.operation, canonical, run); + const page: DesktopRunPage = { + schemaVersion: 1, + runId: run.runId, + terminal, + status: canonical.status, + items: [], + total: items.length, + omissions: { items: items.length - offset, textBytes: 0 }, + }; + for (const item of items.slice(offset, offset + (input.limit ?? 10))) { + const clipped = this.clipItem(item); + page.items.push(clipped.item); + if (Buffer.byteLength(JSON.stringify(page)) > maxResponseBytes - 128) { + page.items.pop(); + break; + } + page.omissions.textBytes += clipped.omitted; + } + page.omissions.items = items.length - page.items.length; + const nextOffset = offset + page.items.length; + if (nextOffset < items.length) { + if (grant.cursors.size >= maxCursors) grant.cursors.delete(grant.cursors.keys().next().value!); + page.nextCursor = randomUUID(); + grant.cursors.set(page.nextCursor, { runId: run.runId, operation: input.operation, digest, offset: nextOffset }); + } + return page; + } + + private redact(text: string): string { + return redactDesktopSecrets(text, this.secrets); + } + + private clipItem(item: DesktopRunQueryItem): { item: DesktopRunQueryItem; omitted: number } { + const title = clip(this.redact(item.title), maxTextBytes); + const message = item.message === undefined ? undefined : clip(this.redact(item.message), maxTextBytes); + return { + item: { + id: item.id, + title: title.text, + ...(item.status ? { status: item.status } : {}), + ...(message ? { message: message.text } : {}), + }, + omitted: title.omitted + (message?.omitted ?? 0), + }; + } + + private items(operation: string, result: DesktopRunResult, run: RegisteredRun): DesktopRunQueryItem[] { + const id = (category: string, value: string) => createHash('sha256').update(`${run.runId}:${category}:${value}`).digest('hex'); + switch (operation) { + case 'tests': + return result.tests.map((test) => ({ + id: id('test', test.testId), + title: test.title, + status: test.status, + message: test.attempts.at(-1)?.failure?.error.message ?? test.attempts.at(-1)?.reason, + })); + case 'work': + return result.work.map((work) => ({ id: id('work', work.workId), title: work.label, status: work.status })); + case 'failures': + return result.failures.map((failure) => ({ + id: id('failure', failure.failureId), + title: failure.kind, + status: 'failed', + message: failure.error.message, + })); + case 'artifacts': + return [...run.artifacts].flatMap(([artifactId, relative]) => { + const artifact = result.artifacts.find((artifact) => artifact.path === relative); + return artifact ? [{ id: artifactId, title: artifact.description }] : []; + }); + default: + throw new TypeError('Invalid desktop run page operation.'); + } + } +} + +export class DesktopRunQueryClient { + private readonly request: (query: DesktopRunQuery) => DesktopRunQueryResponse | Promise; + + constructor(request: (query: DesktopRunQuery) => DesktopRunQueryResponse | Promise) { + this.request = request; + } + + async query(query: DesktopRunQuery): Promise { + validateDesktopRunQuery(query); + return this.request(query); + } +} + +export function validateDesktopRunQuery(value: unknown): asserts value is DesktopRunQuery { + const request = record(value); + opaqueId(request.runId); + const fields = + request.operation === 'status' + ? ['operation', 'runId'] + : request.operation === 'artifact' + ? ['operation', 'runId', 'artifactId'] + : ['operation', 'runId', 'limit', 'cursor']; + if (!['status', 'tests', 'work', 'failures', 'artifacts', 'artifact'].includes(String(request.operation))) + throw new TypeError('Invalid desktop run query operation.'); + if (Object.keys(request).some((key) => !fields.includes(key))) throw new TypeError('Unknown desktop run query field.'); + if (request.operation === 'artifact') opaqueId(request.artifactId); + if (request.cursor !== undefined) opaqueId(request.cursor); + if ( + request.limit !== undefined && + (typeof request.limit !== 'number' || !Number.isSafeInteger(request.limit) || request.limit < 1 || request.limit > 25) + ) + throw new TypeError('Desktop run page limit must be an integer between 1 and 25.'); +} + +function readRetainedText(root: string, relative: string, maximum: number): string { + const file = desktopArtifactPath(root, relative); + const fd = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK); + try { + const stat = fs.fstatSync(fd); + if (!stat.isFile() || stat.nlink !== 1 || stat.size > maximum) + throw new Error('Retained desktop report exceeds its read limit or is not a private regular file.'); + const buffer = Buffer.alloc(Math.min(stat.size + 1, maximum + 1)); + const length = fs.readSync(fd, buffer, 0, buffer.length, 0); + if (length !== stat.size || length > maximum) throw new Error('Retained desktop report changed during reading; retry the query.'); + return new TextDecoder('utf-8', { fatal: true }).decode(buffer.subarray(0, length)); + } finally { + fs.closeSync(fd); + } +} + +function parseResult(text: string, runId: string): DesktopRunResult & { terminal?: boolean } { + let input: unknown; + try { + input = JSON.parse(text); + } catch { + throw new TypeError('Persisted desktop run is not valid JSON.'); + } + const value = record(input); + if ( + value.schemaVersion !== 1 || + value.runId !== runId || + (value.kind !== 'tests' && value.kind !== 'lifecycle') || + (value.terminal !== undefined && typeof value.terminal !== 'boolean') + ) + throw new TypeError('Invalid persisted desktop run identity or status.'); + statusField(value.status); + for (const field of ['tests', 'work', 'failures', 'artifacts']) { + if (!Array.isArray(value[field])) throw new TypeError('Invalid persisted desktop run collection.'); + } + const tests = (value.tests as unknown[]).map((entry) => { + const test = record(entry); + textField(test.testId); + textField(test.title); + statusField(test.status); + if (typeof test.flaky !== 'boolean' || !Array.isArray(test.attempts) || !test.attempts.length) + throw new TypeError('Invalid persisted desktop test attempts.'); + const attempts = test.attempts.map((entry: unknown) => { + const attempt = record(entry); + statusField(attempt.status); + if (typeof attempt.attempt !== 'number' || !Number.isSafeInteger(attempt.attempt) || attempt.attempt < 0) + throw new TypeError('Invalid persisted desktop attempt ordinal.'); + if (typeof attempt.durationMs !== 'number' || !Number.isFinite(attempt.durationMs) || attempt.durationMs < 0) + throw new TypeError('Invalid persisted desktop attempt duration.'); + const reason = attempt.reason; + if (reason !== undefined) textField(reason); + return { + attempt: attempt.attempt, + durationMs: attempt.durationMs, + status: attempt.status, + ...(typeof reason === 'string' ? { reason } : {}), + ...(attempt.failure !== undefined ? { failure: new DesktopReportedError(attempt.failure).failure } : {}), + }; + }); + if (attempts.at(-1)!.status !== test.status) throw new TypeError('Persisted desktop test status disagrees with its latest attempt.'); + return { testId: test.testId, title: test.title, status: test.status, flaky: test.flaky, attempts }; + }); + const work = (value.work as unknown[]).map((entry) => { + const item = record(entry); + textField(item.workId); + textField(item.label); + statusField(item.status); + return { workId: item.workId, label: item.label, status: item.status }; + }); + const failures = (value.failures as unknown[]).map((entry) => new DesktopReportedError(entry).failure); + const artifacts = (value.artifacts as unknown[]).map((entry) => { + const artifact = record(entry); + textField(artifact.path); + textField(artifact.description); + return { path: artifact.path, description: artifact.description }; + }); + textField(value.startedAt); + textField(value.finishedAt); + if (typeof value.durationMs !== 'number' || !Number.isFinite(value.durationMs) || value.durationMs < 0) + throw new TypeError('Invalid persisted desktop run duration.'); + const result = summarizeDesktopRun({ + schemaVersion: 1, + runId, + kind: value.kind, + startedAt: value.startedAt, + finishedAt: value.finishedAt, + durationMs: value.durationMs, + tests, + work, + failures, + artifacts, + }); + return { ...result, status: value.status, ...(typeof value.terminal === 'boolean' ? { terminal: value.terminal } : {}) }; +} + +function record(value: unknown): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) throw new TypeError('Expected a desktop run query object.'); + return value as Record; +} +function opaqueId(value: unknown): asserts value is string { + if (typeof value !== 'string' || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(value)) + throw new TypeError('Expected an opaque desktop run identifier.'); +} +function deadline(value: number): void { + if (!Number.isSafeInteger(value) || value <= Date.now()) + throw new TypeError('Desktop run retention/grant deadline must be in the future.'); +} +function textField(value: unknown): asserts value is string { + if (typeof value !== 'string') throw new TypeError('Invalid persisted desktop report text.'); +} +function statusField(value: unknown): asserts value is TestStatus { + if (!reportStatuses.includes(value as TestStatus)) throw new TypeError('Invalid persisted desktop report status.'); +} +function clip(text: string, maximum: number): { text: string; omitted: number } { + const bytes = Buffer.byteLength(text); + if (bytes <= maximum) return { text, omitted: 0 }; + let retained = ''; + let size = 0; + for (const character of text) { + const length = Buffer.byteLength(character); + if (size + length > maximum) break; + retained += character; + size += length; + } + return { text: retained, omitted: bytes - size }; +}