From fb28eac82270c837d45e47d6b1cc03db60be36f1 Mon Sep 17 00:00:00 2001 From: Jason Morse Date: Thu, 12 Nov 2026 16:16:39 +0000 Subject: [PATCH 1/5] Attribute desktop story case timing and lifecycle observations Part of #4435. Record bounded local selection, worker, cleanup and native-activity diagnostics without changing navigation or remount isolation. Capture the eight-run Fabric baseline and its observation limits. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .changeset/calm-story-observations.md | 6 + .../references/current-status.md | 26 +++ .../desktop-driver/references/protocol.md | 10 ++ .../src/server/sessionStories.ts | 42 ++++- .../src/server/storyOperations.test.ts | 24 +++ .../agentic/desktop-driver/src/storybook.ts | 16 ++ packages/agentic/storybook-desktop/README.md | 3 + .../storybook-desktop/references/README.md | 1 + .../references/current-status.md | 87 ++++++++++ .../references/story-tests.md | 39 +++++ .../StorybookChannelOrchestrator.test.ts | 35 ++++ .../driver/StorybookChannelOrchestrator.ts | 38 +++- .../src/driver/serverIntegration.test.ts | 5 + .../src/testing/runDiagnostics.test.ts | 91 ++++++++++ .../src/testing/runDiagnostics.ts | 112 ++++++++++++ .../src/testing/runWdioTests.ts | 162 +++++++++++++----- .../src/testing/wdioRunner.contract.cjs | 17 ++ .../storybook-desktop/src/testing/worker.ts | 50 ++++-- 18 files changed, 695 insertions(+), 69 deletions(-) create mode 100644 .changeset/calm-story-observations.md create mode 100644 packages/agentic/storybook-desktop/references/current-status.md create mode 100644 packages/agentic/storybook-desktop/src/testing/runDiagnostics.test.ts create mode 100644 packages/agentic/storybook-desktop/src/testing/runDiagnostics.ts diff --git a/.changeset/calm-story-observations.md b/.changeset/calm-story-observations.md new file mode 100644 index 0000000000..a8effbc392 --- /dev/null +++ b/.changeset/calm-story-observations.md @@ -0,0 +1,6 @@ +--- +'@fluentui-react-native/desktop-driver': patch +'@fluentui-react-native/storybook-desktop': patch +--- + +Add bounded, service-local story selection timings and per-case execution/lifecycle observations without changing authenticated readiness, preview remounts, or worker/session isolation. Part of #4435. diff --git a/packages/agentic/desktop-driver/references/current-status.md b/packages/agentic/desktop-driver/references/current-status.md index 507829248c..12b95163e2 100644 --- a/packages/agentic/desktop-driver/references/current-status.md +++ b/packages/agentic/desktop-driver/references/current-status.md @@ -472,3 +472,29 @@ after the disconnect, so fresh-preview fencing remains Partial. The first probe timed out after 10 seconds; bridge retirement occurred about 19 seconds after injection. A later attempt waited for retirement and retained the same Partial result. Physical Ctrl+C and DPI changes were not exercised. + +## 2026-11-12: bounded refresh timing observations + +Part of #4435. Story selection can return allowlisted, service-local timing +observations and a canonical native-verified PID/start-time pair. They are +response diagnostics, not authenticated marker fields, context identity, or +authority to clear quarantine. Request/run/epoch matching and fresh native +marker verification remain unchanged; provider payloads and private values +are not copied into these observations. + +The instrumented baseline ran eight independently launched macOS Fabric +accessible cohorts, each with three passing ListboxItem callbacks and one +declared macOS-unsupported callback. Each passing callback selected a fresh +keyed preview/context generation; every cohort retained one verified app +incarnation and one authenticated bridge connection during its callbacks. +JS initialization/HMR was not observed independently and cannot be inferred +from hello counts. No activation or main/focused-window change was observed in +this observational cohort; a zero advisory count is not visual qualification. +The Storybook evidence ledger, +`packages/agentic/storybook-desktop/references/current-status.md`, records +selection, timing, variability, and limitations. + +This timing implementation changes no Swift source, physical-input +activation, foreground checks, hit-testing, focus, or window-switch policy. +It does not qualify those paths or remove provider-local checks. Windows +Fabric and Win32 Paper were **not run locally** for this change. diff --git a/packages/agentic/desktop-driver/references/protocol.md b/packages/agentic/desktop-driver/references/protocol.md index 01b62dc7fb..c31184bb08 100644 --- a/packages/agentic/desktop-driver/references/protocol.md +++ b/packages/agentic/desktop-driver/references/protocol.md @@ -228,6 +228,16 @@ detached root reports `stale element reference`. ## Storybook readiness +Part of #4435. Successful selection/reset responses may include `diagnostics` +with fixed numeric preparation, HTTP completion, correlated-ready, +native-verification and marker-poll observations, authenticated hello/connection +counts, and the native-verified application PID/start time. Providers fill an +optional local `StoryProviderContext.diagnostics`; it is never sent to the +runtime. The service allowlists finite nonnegative numbers before returning +them. These observations are not authenticated marker fields, are not part of +context identity, and never authorize readiness or recovery. In particular, +hello counts are not JavaScript initialization counts. + When a target provides a `StoryOrchestrator`, the consumer validates: - instance, target, endpoint, and catalog identity; diff --git a/packages/agentic/desktop-driver/src/server/sessionStories.ts b/packages/agentic/desktop-driver/src/server/sessionStories.ts index 78a591ec62..67750e1c22 100644 --- a/packages/agentic/desktop-driver/src/server/sessionStories.ts +++ b/packages/agentic/desktop-driver/src/server/sessionStories.ts @@ -5,7 +5,7 @@ import type { OperationCurrentStory, OperationStories } from '../operations/runt import { HostStaleError, WebDriverError } from '../protocol/errors.js'; import { attachFailureEffect, readFailureEffect } from '../protocol/settlement.js'; import { withCommandTimeout } from '../protocol/timeouts.js'; -import type { StoryOrchestrator, StoryReadyResult, StorySelectionRequest } from '../storybook.js'; +import type { StoryOrchestrator, StoryReadyResult, StorySelectionDiagnostics, StorySelectionRequest } from '../storybook.js'; import { matchesStoryMarker, parseStoryMarker, StoryBridgeUnavailableError } from '../storybook.js'; import type { DesktopSession, SessionManager } from './SessionManager.js'; @@ -32,7 +32,8 @@ export async function changeStory( try { return await withCommandTimeout( async (signal) => { - const context = { lifetimeId: session.id, signal }; + const diagnostics: StorySelectionDiagnostics = {}; + const context = { lifetimeId: session.id, signal, diagnostics }; const result = await (change === 'reset' ? orchestrator.resetStory(request, context) : orchestrator.selectStory(request, context)); @@ -43,11 +44,36 @@ export async function changeStory( ) { throw new WebDriverError('unknown error', 'Storybook returned readiness for a different selection.'); } - const verified = await verifyStoryMarker(session, result, signal); + const started = performance.now(); + const verified = await verifyStoryMarker(session, result, signal, diagnostics); + diagnostics.nativeVerificationMs = performance.now() - started; signal.throwIfAborted(); unavailableContexts.delete(session); session.story = result; - return { result, verified }; + const observations: NonNullable = {}; + for (const key of [ + 'preparationMs', + 'httpMs', + 'runtimeReadyMs', + 'nativeVerificationMs', + 'nativeMarkerPolls', + 'authenticatedHellos', + 'bridgeConnections', + ] as const) { + const value = diagnostics[key]; + if (typeof value === 'number' && Number.isFinite(value) && value >= 0) observations[key] = value; + } + const { processId, processStartedAt } = session.lease; + if ( + Number.isSafeInteger(processId) && + processId! > 0 && + processStartedAt && + processStartedAt.length <= 64 && + Number.isFinite(Date.parse(processStartedAt)) + ) { + observations.application = { processId: processId!, processStartedAt: new Date(processStartedAt).toISOString() }; + } + return { result: { ...result, diagnostics: observations }, verified }; }, session.desktopTimeouts.storyRender, 'Changing Storybook context', @@ -152,7 +178,12 @@ export async function assertStoryContext(session: DesktopSession, sessions: Sess } /** Polls the story root until it shows `expected`. Returns false when the target has no native story root. */ -async function verifyStoryMarker(session: DesktopSession, expected: StoryReadyResult, signal: AbortSignal): Promise { +async function verifyStoryMarker( + session: DesktopSession, + expected: StoryReadyResult, + signal: AbortSignal, + diagnostics?: StorySelectionDiagnostics, +): Promise { const testId = session.target.storyRootTestId; if (!testId) { if (expected.markerVersion === 1) { @@ -162,6 +193,7 @@ async function verifyStoryMarker(session: DesktopSession, expected: StoryReadyRe } for (;;) { signal.throwIfAborted(); + if (diagnostics) diagnostics.nativeMarkerPolls = (diagnostics.nativeMarkerPolls ?? 0) + 1; const markers = await readMarkers(session, testId, signal); if (markers.some((marker) => matchesStoryMarker(marker, expected))) { return true; diff --git a/packages/agentic/desktop-driver/src/server/storyOperations.test.ts b/packages/agentic/desktop-driver/src/server/storyOperations.test.ts index c9edf053db..51f9a73eb2 100644 --- a/packages/agentic/desktop-driver/src/server/storyOperations.test.ts +++ b/packages/agentic/desktop-driver/src/server/storyOperations.test.ts @@ -53,6 +53,30 @@ async function withStories(body: (fixture: Fixture) => Promise, options: F } describe('story operations', () => { + test('selection reports local phase timing and the verified app incarnation without serializing provider context', async () => { + await withStories(async ({ harness, session }) => { + const state = harness.server.sessions.get(session.id); + state.lease.processId = 123; + state.lease.processStartedAt = '2026-11-12T00:00:00.000Z'; + const select = harness.storyOrchestrator!.selectStory.bind(harness.storyOrchestrator); + harness.storyOrchestrator!.selectStory = async (request, context) => { + Object.assign(context?.diagnostics ?? {}, { preparationMs: 2, runtimeReadyMs: 3, nonce: 'never-report-this' }); + return select(request, context); + }; + const ready = await session.selectStory(manifest.entries[0].id); + expect(ready.diagnostics).toMatchObject({ + preparationMs: 2, + runtimeReadyMs: 3, + nativeVerificationMs: expect.any(Number), + nativeMarkerPolls: 1, + application: { processId: 123, processStartedAt: '2026-11-12T00:00:00.000Z' }, + }); + expect(JSON.stringify(ready)).not.toContain('never-report-this'); + expect(ready).not.toHaveProperty('signal'); + expect(ready).not.toHaveProperty('lifetimeId'); + }); + }); + test('built-in manifest, current, reset and args routes all propagate local provider context', async () => { await withStories(async ({ harness, session }) => { const orchestrator = harness.storyOrchestrator!; diff --git a/packages/agentic/desktop-driver/src/storybook.ts b/packages/agentic/desktop-driver/src/storybook.ts index 7467788996..a55da3c012 100644 --- a/packages/agentic/desktop-driver/src/storybook.ts +++ b/packages/agentic/desktop-driver/src/storybook.ts @@ -41,12 +41,28 @@ export type StoryReadyResult = { requestId?: string; runId: string; storyId: string; + /** Service-local observations, not part of the authenticated runtime marker or readiness decision. */ + diagnostics?: StorySelectionDiagnostics & { + application?: { processId: number; processStartedAt: string }; + }; +}; + +export type StorySelectionDiagnostics = { + preparationMs?: number; + httpMs?: number; + runtimeReadyMs?: number; + nativeVerificationMs?: number; + nativeMarkerPolls?: number; + authenticatedHellos?: number; + bridgeConnections?: number; }; /** Local provider lifetime; never included in channel messages or serialized operation inputs. */ export type StoryProviderContext = { lifetimeId: string; signal: AbortSignal; + /** Optional, fixed numeric observations. Never include runtime payloads or credentials here. */ + diagnostics?: StorySelectionDiagnostics; }; export class StoryProviderError extends Error { diff --git a/packages/agentic/storybook-desktop/README.md b/packages/agentic/storybook-desktop/README.md index f039e62b04..0c4f036dd3 100644 --- a/packages/agentic/storybook-desktop/README.md +++ b/packages/agentic/storybook-desktop/README.md @@ -23,6 +23,9 @@ Read [integration](references/integration.md) for app wiring and [story-tests](references/story-tests.md) to author inline `wdio` callbacks. Use `--reporter json` for bounded machine-readable summaries or NDJSON for live progress. +Executable story artifacts include bounded phase/lifecycle observations; +see [story-test diagnostics](references/story-tests.md#timing-and-lifecycle-observations). +Part of #4435. Agent-served channels reject alternate HTTP mutations and never relay client WebSocket frames; selection/reset go through the session-owning driver grant, and args remain unsupported. Non-agent channels are explicitly legacy diff --git a/packages/agentic/storybook-desktop/references/README.md b/packages/agentic/storybook-desktop/references/README.md index 2e0708a4c8..0734ff1a4c 100644 --- a/packages/agentic/storybook-desktop/references/README.md +++ b/packages/agentic/storybook-desktop/references/README.md @@ -18,3 +18,4 @@ owner_paths: | `storybook-desktop.win32` | [Win32](platforms/win32.md) | Bundling or running the REX host | | `storybook-desktop.troubleshooting` | [Troubleshooting](troubleshooting.md) | Diagnosing build, extraction, readiness, or smoke failures | | `storybook-desktop.contributing` | [Contributing](contributing.md) | Editing package code or tests | +| `storybook-desktop.current-status` | [Current evidence](current-status.md) | Checking bounded measurements and qualification gaps | diff --git a/packages/agentic/storybook-desktop/references/current-status.md b/packages/agentic/storybook-desktop/references/current-status.md new file mode 100644 index 0000000000..7c1d2956b3 --- /dev/null +++ b/packages/agentic/storybook-desktop/references/current-status.md @@ -0,0 +1,87 @@ +--- +id: storybook-desktop.current-status +kind: instruction-reference +owner_paths: + - packages/agentic/storybook-desktop +--- + +# Current qualification evidence + +## 2026-11-12: refresh-reduction timing baseline + +Part of #4435. This section records bounded measurements, not a reliability +claim or a full-catalog test pass. The product baseline is the instrumented +parent of the navigation optimization, stacked on PR #4490 +(`aadc2f3591d15f5d43496d7f069fa8d0d32d5384`). + +### Method and coverage + +The public `DesktopStorybookCli` API ran eight independently launched macOS +Fabric lifecycles in the accessible lane. An injected fetch narrowed only the +CLI's traversal index to the real +`components-listboxitem--native-selection` entry; the server's authenticated +manifest, runtime, native helper, and authored callbacks were unchanged. +Every lifecycle traversed that story, ran its three supported callbacks in +declaration order, and reported its one declared macOS-unsupported callback. +App, channel, helper, and Metro cleanup completed before the next lifecycle. +This deliberately bounded index is **not** complete smoke traversal. + +The measurements below are from the executable-story `run.json`, excluding +application/service startup and traversal. Millisecond p50 is the median; p95 +uses inclusive linear interpolation. Phase samples are 24 executed callbacks; +runner-total samples are eight lifecycles. Navigation subphases overlap and +must not be added together. The first lifecycle can include cold caches. + +| Fabric baseline phase | p50 ms | p95 ms | +| ---------------------------- | ------: | ------: | +| Session attach | 1192.22 | 1612.56 | +| Manifest | 2.62 | 3.37 | +| Input policy | 0.01 | 0.04 | +| Preparation | 0.13 | 0.17 | +| Upstream HTTP | 503.23 | 504.56 | +| Correlated runtime readiness | 28.77 | 562.62 | +| Native marker verification | 75.07 | 168.00 | +| Complete navigation | 753.13 | 812.97 | +| Worker startup | 203.00 | 215.85 | +| Worker body | 505.18 | 533.74 | +| Settled worker | 723.34 | 750.74 | +| Receipts | 2.62 | 2.93 | +| Session cleanup | 115.24 | 126.77 | +| Diagnostic native reads | 13.33 | 18.08 | + +Runner-total p50/p95 were **7939.07 / 8845.37 ms**; aggregate duration was +**63950.71 ms**. Sample variance was **306828.83 ms²** (range +7478.77–9245.48 ms). Complete lifecycle p50/p95, including startup and +traversal, were 19887.17 / 28433.88 ms, with 171994.83 ms aggregate and +19950017.41 ms² sample variance. These different totals are not interchangeable. + +### Lifecycle and observation limits + +Each lifecycle had one verified PID/start-time pair, one authenticated bridge +connection, and one accepted hello. Each callback selected a fresh generation +(2, 3, 4 after the traversal's first generation), with matching keyed-mount and +context generations. No within-lifecycle application replacement or new bridge +connection was observed. JS initialization/HMR is explicitly **not observed**: +neither a hello nor the absence of a new socket proves a JS initialization count. +The keyed generation is not an instrumented React effect-mount counter. + +Native activity reports retained fixed counters only. No activation command, +provider activation, application activation/deactivation, or main/focused-window +change was observed in this observational cohort. `AXUIElementDestroyed` +notifications can describe preview elements, not necessarily closed windows. +The bounded native event ring can be truncated, and its observations are +advisory. These counters do not establish whether a user saw a flash. + +Target-window-only ScreenCaptureKit recordings remain local in ignored +consumer artifacts. They are separate evidence, not committed media. +Paired candidate/renderer evidence and visual analysis must be recorded before +claiming a speedup or flash reduction. + +### Gates + +Session attachment is a material, dominant cost in this cohort; the conditional +session/worker investigation gate is eligible. No session reuse, worker +sharing, worker-isolation flag, preview reuse, or runtime protocol expansion is +implemented by the timing work. Such a trial still needs independently paired +benefit and ownership/settlement evidence. Windows Fabric and Win32 Paper: +**not run locally**. diff --git a/packages/agentic/storybook-desktop/references/story-tests.md b/packages/agentic/storybook-desktop/references/story-tests.md index 2f4b5a3025..20df61f1a1 100644 --- a/packages/agentic/storybook-desktop/references/story-tests.md +++ b/packages/agentic/storybook-desktop/references/story-tests.md @@ -161,6 +161,45 @@ editing callbacks because their digest participates in manifest freshness. filters names. `test` operates against an already running app/driver; smoke owns the complete app lifecycle. +### Timing and lifecycle observations + +Part of #4435. The linked **Executable story results** `run.json` artifact +records `diagnostics` without changing test selection, readiness, or isolation. +Run-level fields are `extractionMs`, `sourceExtractions`, and `totalMs`. +Each attempted case records `diagnostics.phases`: + +- `sessionAttachMs`, `manifestMs`, and `policyMs`; +- `navigationMs`, with service-local `preparationMs`, optional `httpMs`, + `runtimeReadyMs`, and `nativeVerificationMs`; +- settled `workerMs`, `workerStartupMs`, `workerBodyMs`, and + `workerSettlementMs` when the worker supplied a result; +- `receiptsMs`, settled `cleanupMs`, and `activityReadMs`. + +These are observations, not deadlines or success evidence. Navigation +subphases overlap: HTTP completion and runtime readiness run concurrently. +Readiness is elapsed from the start of preparation; HTTP timing starts at its +dispatch. Do not sum those fields. Worker startup includes spec preparation, +Node/import startup, reporter setup, and attachment; body timing surrounds the +callback. Settlement is the remaining supervisor-observed worker time, including +process exit and stop. Missing fields mean the phase did not complete or its +producer could not observe it, never a zero-cost pass. + +`diagnostics.lifecycle` records the context and mount generations (equal under +the default keyed remount), native marker poll count, authenticated hello and +connection counters, and the native-verified application PID/start time when +available. `jsInitializations: 'not-observed'` is explicit: a bridge hello can +answer another challenge without initializing JavaScript. No runtime message, +descriptor, or preview-reuse protocol is added. + +`windowActivity` reduces the existing native diagnostic rings to per-case +notification and activation-command counters. It never retains native handles, +window titles, argument values, or raw command details. Unsupported or failed +reads are explicit. `observing`, registration-failure counts, and truncation +flags limit interpretation: these are advisory lower bounds, not proof of +foreground ownership or visible flashing. Target-window-only recording is a +separate, permission-gated qualification; no recording is enabled by the runner. +The observation reads themselves are included in `activityReadMs`. + ## Run an input lane A lane runs only the cases that apply to one way of driving the app. The diff --git a/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.test.ts b/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.test.ts index 778b114d46..b4aefaaa3e 100644 --- a/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.test.ts +++ b/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.test.ts @@ -426,6 +426,41 @@ describe('durable authenticated bridge epochs', () => { }); describe('StorybookChannelOrchestrator epochs and lifetime', () => { + test('attributes HTTP and correlated readiness separately without treating repeated hellos as JS initializations', async () => { + jest.useFakeTimers(); + try { + let finish!: (response: Response) => void; + const { client, orchestrator } = setup( + jest.fn(() => new Promise((resolve) => (finish = resolve))), + 1000, + ); + client.authenticate(); + client.hello(); + const diagnostics = {}; + const selection = orchestrator.selectStory(request, { + lifetimeId: 'session', + signal: new AbortController().signal, + diagnostics, + }); + await jest.advanceTimersByTimeAsync(20); + client.ready(); + await jest.advanceTimersByTimeAsync(80); + finish(new Response('{}')); + await selection; + expect(diagnostics).toMatchObject({ + preparationMs: 0, + httpMs: 100, + runtimeReadyMs: 20, + authenticatedHellos: 2, + bridgeConnections: 1, + }); + expect(diagnostics).not.toHaveProperty('jsInitializations'); + expect(JSON.stringify(diagnostics)).not.toContain(manifest.bridgeNonce); + } finally { + jest.useRealTimers(); + } + }); + test('absent and hello-only bridges reject before dispatch with no effect', async () => { const { client, fetch, orchestrator } = setup(); await expect(orchestrator.selectStory(request)).rejects.toMatchObject({ diff --git a/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.ts b/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.ts index 2b158a00f6..e20fb1bc8c 100644 --- a/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.ts +++ b/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.ts @@ -58,12 +58,15 @@ export class StorybookChannelOrchestrator implements StoryOrchestrator { private readonly initializedClients = new WeakSet(); private readonly fencedClients = new WeakSet(); private readonly recordedClients = new WeakSet(); + private readonly authenticatedClients = new WeakSet(); private readonly recordings = new WeakMap>(); private readonly journal?: BridgeEpochJournal; private bridgeClient?: ChannelClient; private bridgeConnected = false; private needsRelaunch = false; private selectionSequence = 0; + private authenticatedHellos = 0; + private bridgeConnections = 0; private currentStory: StoryReadyResult | null = null; constructor({ @@ -113,6 +116,7 @@ export class StorybookChannelOrchestrator implements StoryOrchestrator { } private async selectAndPrepareStory(request: StorySelectionRequest, context?: StoryProviderContext): Promise { + const started = performance.now(); this.requireStory(request.storyId); if (context?.signal.aborted) throw new StoryProviderError('Story selection was cancelled before dispatch.', 'none'); if (this.journal) await this.checkBridgeRecovery(); @@ -137,6 +141,9 @@ export class StorybookChannelOrchestrator implements StoryOrchestrator { void failure.catch(() => undefined); const ready = new Promise((resolve) => { this.pending.set(request.requestId, { epoch, reject: stop, request, resolve }); + }).then((result) => { + if (context?.diagnostics) context.diagnostics.runtimeReadyMs = performance.now() - started; + return result; }); const onAbort = () => stop(new Error('Story selection was cancelled.')); context?.signal.addEventListener('abort', onAbort, { once: true }); @@ -150,19 +157,29 @@ export class StorybookChannelOrchestrator implements StoryOrchestrator { args: [{ ...request, connectionEpoch: epoch, selectionSequence: ++this.selectionSequence }], }), ); + if (context?.diagnostics) { + context.diagnostics.preparationMs = performance.now() - started; + context.diagnostics.authenticatedHellos = this.authenticatedHellos; + context.diagnostics.bridgeConnections = this.bridgeConnections; + } + const httpStarted = performance.now(); const navigation = this.controlled ? Promise.resolve(client.send(JSON.stringify({ type: 'setCurrentStory', args: [{ storyId: request.storyId, viewMode: 'story' }] }))) : this.fetch(`${this.serverUrl}/select-story-sync/${encodeURIComponent(request.storyId)}`, { method: 'POST', signal: controller.signal, - }).then(async (response) => { - if (!response.ok && response.status !== 408) { - const detail = await response.text(); - throw new Error( - `Storybook failed to select "${request.storyId}" with status ${response.status}${detail ? `: ${detail}` : '.'}`, - ); - } - }); + }) + .then(async (response) => { + if (!response.ok && response.status !== 408) { + const detail = await response.text(); + throw new Error( + `Storybook failed to select "${request.storyId}" with status ${response.status}${detail ? `: ${detail}` : '.'}`, + ); + } + }) + .finally(() => { + if (context?.diagnostics) context.diagnostics.httpMs = performance.now() - httpStarted; + }); const [, result] = await Promise.race([Promise.all([navigation, ready]), failure]); this.currentStory = result; return result; @@ -265,6 +282,11 @@ export class StorybookChannelOrchestrator implements StoryOrchestrator { payload.nonce !== expected.bridgeNonce ) return; + this.authenticatedHellos++; + if (!this.authenticatedClients.has(client)) { + this.authenticatedClients.add(client); + this.bridgeConnections++; + } // A runtime reload can authenticate its new socket before the old socket reports close; the old context is lost either way. if (this.bridgeClient && this.bridgeClient !== client) this.retireBridge(this.bridgeClient, 'The authenticated Storybook runtime bridge was replaced by a new runtime connection.'); diff --git a/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts b/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts index 9c1c338674..9d8bf8d40a 100644 --- a/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts +++ b/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts @@ -148,6 +148,11 @@ describe('desktop Storybook server integration', () => { }, }); await expect(session.selectStory('components-button--default', 'integration-run')).resolves.toEqual({ + diagnostics: { + application: { processId: 1000, processStartedAt: '1970-01-01T00:00:00.000Z' }, + nativeMarkerPolls: 1, + nativeVerificationMs: expect.any(Number), + }, previewGeneration: 1, runId: 'integration-run', storyId: 'components-button--default', diff --git a/packages/agentic/storybook-desktop/src/testing/runDiagnostics.test.ts b/packages/agentic/storybook-desktop/src/testing/runDiagnostics.test.ts new file mode 100644 index 0000000000..399dfcde67 --- /dev/null +++ b/packages/agentic/storybook-desktop/src/testing/runDiagnostics.test.ts @@ -0,0 +1,91 @@ +import { readApplicationIncarnation, readWorkerTimings, summarizeWindowActivity } from './runDiagnostics.js'; + +test('projects only a bounded application incarnation without retaining injected context', () => { + const application = { processId: 123, processStartedAt: '2026-11-12T12:00:00.000Z' }; + expect(readApplicationIncarnation({ ...application, nonce: 'private', args: { secret: true } })).toEqual(application); + expect(readApplicationIncarnation({ ...application, processStartedAt: 'Thu Nov 12 2026 12:00:00 GMT+0000 (private)' })).toEqual( + application, + ); + for (const invalid of [ + { ...application, processId: -1 }, + { ...application, processId: 0.5 }, + { ...application, processStartedAt: 'private' }, + { ...application, processStartedAt: ' '.repeat(100) }, + null, + ]) { + expect(readApplicationIncarnation(invalid)).toBeUndefined(); + } +}); + +test('reduces native activity to bounded counters, never window titles or command details', () => { + const activity = summarizeWindowActivity({ + source: 'AX', + observing: true, + eventsTruncated: true, + events: [ + { ageMs: 1, element: 'private-native-handle', notification: 'AXMainWindowChanged', sequence: 2 }, + { ageMs: 0, element: 'private-native-handle', notification: 'AXWindowCreated', sequence: 3 }, + ], + commands: [ + { ageMs: 3, command: 'activate', outcome: 'ok', sequence: 5, detail: { method: 'already-frontmost', name: 'private text' } }, + { ageMs: 2, command: 'click', outcome: 'ok', sequence: 6, detail: { method: 'accessibility', args: 'private args' } }, + { ageMs: 1, command: 'find', outcome: 'ok', sequence: 7 }, + ], + trackedWindows: [{ closed: false, element: 'private-handle', id: 'private-window', primary: true, title: 'private title' }], + }); + expect(activity).toEqual({ + status: 'observed', + events: 2, + observing: true, + notificationRegistrationFailures: 0, + applicationActivations: 0, + applicationDeactivations: 0, + focusedWindowChanges: 0, + mainWindowChanges: 1, + windowsCreated: 1, + elementsDestroyed: 0, + activationCommands: 1, + providerActivations: 2, + commandsTruncated: true, + eventsTruncated: true, + }); + expect(JSON.stringify(activity)).not.toContain('private'); +}); + +test('activity counters use per-case sequence deltas, preserving advisory truncation', () => { + expect( + summarizeWindowActivity( + { + source: 'AX', + observing: true, + eventsTruncated: false, + events: [ + { ageMs: 1, element: 'window', notification: 'AXMainWindowChanged', sequence: 10 }, + { ageMs: 0, element: 'window', notification: 'AXUIElementDestroyed', sequence: 11 }, + ], + commands: [ + { ageMs: 2, command: 'activate', outcome: 'ok', sequence: 20 }, + { ageMs: 1, command: 'find', outcome: 'ok', sequence: 21 }, + ], + }, + { command: 20, event: 10 }, + ), + ).toMatchObject({ + events: 1, + mainWindowChanges: 0, + elementsDestroyed: 1, + activationCommands: 0, + commandsTruncated: false, + eventsTruncated: false, + }); +}); + +test('worker timing projection rejects nonnumeric and unbounded values without retaining extra fields', () => { + expect(readWorkerTimings({ workerStartupMs: 10, workerBodyMs: 20, nonce: 'private' })).toEqual({ + workerStartupMs: 10, + workerBodyMs: 20, + }); + for (const workerBodyMs of [-1, Infinity, NaN, 2_147_483_648, 'private']) { + expect(readWorkerTimings({ workerBodyMs })).toEqual({}); + } +}); diff --git a/packages/agentic/storybook-desktop/src/testing/runDiagnostics.ts b/packages/agentic/storybook-desktop/src/testing/runDiagnostics.ts new file mode 100644 index 0000000000..e1a8b44aab --- /dev/null +++ b/packages/agentic/storybook-desktop/src/testing/runDiagnostics.ts @@ -0,0 +1,112 @@ +import type { DesktopSessionClient } from '@fluentui-react-native/desktop-driver/client'; + +type WindowLog = Awaited>; +export type WdioCaseDiagnostics = { + phases: Partial< + Record< + | 'sessionAttachMs' + | 'manifestMs' + | 'policyMs' + | 'navigationMs' + | 'preparationMs' + | 'httpMs' + | 'runtimeReadyMs' + | 'nativeVerificationMs' + | 'workerMs' + | 'workerStartupMs' + | 'workerBodyMs' + | 'workerSettlementMs' + | 'receiptsMs' + | 'cleanupMs' + | 'activityReadMs', + number + > + >; + lifecycle?: { + contextGeneration: number; + mountGeneration: number; + nativeMarkerPolls?: number; + authenticatedHellos?: number; + bridgeConnections?: number; + /** A hello answers a challenge, not necessarily a JS initialization. No runtime protocol is added to observe it. */ + jsInitializations: 'not-observed'; + application?: { processId: number; processStartedAt: string }; + }; + windowActivity?: WindowActivity; +}; + +export type WdioWorkerTimings = { workerStartupMs?: number; workerBodyMs?: number }; + +export function readApplicationIncarnation(value: unknown): { processId: number; processStartedAt: string } | undefined { + if (!value || typeof value !== 'object') return undefined; + const { processId, processStartedAt } = value as Record; + if ( + typeof processId !== 'number' || + !Number.isSafeInteger(processId) || + processId <= 0 || + typeof processStartedAt !== 'string' || + processStartedAt.length > 64 || + !Number.isFinite(Date.parse(processStartedAt)) + ) { + return undefined; + } + return { processId, processStartedAt: new Date(processStartedAt).toISOString() }; +} + +export function readWorkerTimings(value: unknown): WdioWorkerTimings { + const timings: WdioWorkerTimings = {}; + if (!value || typeof value !== 'object') return timings; + for (const key of ['workerStartupMs', 'workerBodyMs'] as const) { + const duration = (value as Record)[key]; + if (typeof duration === 'number' && Number.isFinite(duration) && duration >= 0 && duration <= 2_147_483_647) timings[key] = duration; + } + return timings; +} + +export type WindowActivityCursor = { command: number; event: number }; +export type WindowActivity = + | { status: 'not-supported' | 'unavailable' } + | { + status: 'observed'; + events: number; + observing: boolean; + notificationRegistrationFailures: number; + applicationActivations: number; + applicationDeactivations: number; + focusedWindowChanges: number; + mainWindowChanges: number; + windowsCreated: number; + elementsDestroyed: number; + activationCommands: number; + providerActivations: number; + commandsTruncated: boolean; + eventsTruncated: boolean; + }; + +export function windowActivityCursor(log: WindowLog): WindowActivityCursor { + return { command: log.commands?.at(-1)?.sequence ?? 0, event: log.events.at(-1)?.sequence ?? 0 }; +} + +/** Project only fixed counters; the native log also contains application text and must not enter the run report. */ +export function summarizeWindowActivity(log: WindowLog, since: WindowActivityCursor = { command: 0, event: 0 }): WindowActivity { + const commands = (log.commands ?? []).slice(-1024).filter(({ sequence }) => sequence > since.command); + const events = log.events.slice(-1024).filter(({ sequence }) => sequence > since.event); + return { + status: 'observed', + events: events.length, + observing: log.observing, + notificationRegistrationFailures: log.registrationFailures?.length ?? 0, + applicationActivations: events.filter(({ notification }) => notification === 'AXApplicationActivated').length, + applicationDeactivations: events.filter(({ notification }) => notification === 'AXApplicationDeactivated').length, + focusedWindowChanges: events.filter(({ notification }) => notification === 'AXFocusedWindowChanged').length, + mainWindowChanges: events.filter(({ notification }) => notification === 'AXMainWindowChanged').length, + windowsCreated: events.filter(({ notification }) => notification === 'AXWindowCreated').length, + elementsDestroyed: events.filter(({ notification }) => notification === 'AXUIElementDestroyed').length, + activationCommands: commands.filter(({ command }) => command === 'activate').length, + providerActivations: commands.filter(({ detail }) => + ['already-frontmost', 'accessibility', 'workspace', 'accessibility+workspace'].includes(String(detail?.method)), + ).length, + commandsTruncated: commands.length > 0 && commands[0].sequence > since.command + 1, + eventsTruncated: log.eventsTruncated || (events.length > 0 && events[0].sequence > since.event + 1), + }; +} diff --git a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts index 87c5941294..2145228d94 100644 --- a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts +++ b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts @@ -22,6 +22,15 @@ import { resolveWdioOptions, type DesktopStorybookWdioOptions } from '../config/ import { NodeDesktopCommandRunner, type DesktopCommandRunner, type PreparedDesktopCommand } from '../cli/commandRunner.js'; import { extractWdioTests } from './extractWdioTests.js'; import { assessWdioCohort } from './laneVerdict.js'; +import { + readApplicationIncarnation, + readWorkerTimings, + summarizeWindowActivity, + windowActivityCursor, + type WdioCaseDiagnostics, + type WdioWorkerTimings, + type WindowActivityCursor, +} from './runDiagnostics.js'; import { csfTestId, decideWdioSelection, @@ -45,6 +54,7 @@ export type WdioStoryTestResult = { input?: TestInputReport; evidenceErrors?: string[]; artifacts?: readonly DesktopArtifact[]; + diagnostics?: WdioCaseDiagnostics; }; export type RunWdioStoryTestsOptions = DesktopStorybookWdioOptions & { @@ -113,6 +123,7 @@ export async function runWdioStoryTests( options: RunWdioStoryTestsOptions, runner?: DesktopCommandRunner, ): Promise { + const runStarted = performance.now(); const settings = resolveWdioOptions(options); const selection = decideWdioSelection(options.manifest, settings, { platform: options.platform, scope: options.scope }); const planned = plannedWdioWork(selection, settings); @@ -125,8 +136,11 @@ export async function runWdioStoryTests( if (selection.stories.length === 0) { throw new Error(`No executable wdio story tests matched the requested selection${describeWdioExclusions(selection)}.`); } + const extractionStarted = performance.now(); + let sourceExtractions = 0; const cases = selection.stories.flatMap((entry) => { const sourceFile = path.join(options.config.resolvePackage(entry.packageName).root, entry.sourcePath); + sourceExtractions++; const extracted = extractWdioTests(fs.readFileSync(sourceFile, 'utf8'), sourceFile).get(entry.wdio!.exportName); if (!extracted || extracted.digest !== entry.wdio!.digest) { throw new Error(`Executable test source changed for ${entry.id}; regenerate the Storybook manifest.`); @@ -140,6 +154,7 @@ export async function runWdioStoryTests( return [{ entry, code, testName: name, descriptor: decision?.descriptor, gap: decision?.gap }]; }); }); + const extractionMs = performance.now() - extractionStarted; if (cases.length === 0) throw new Error('No executable wdio tests matched the requested selection.'); const plannedIds = new Set(planned.map(({ workId }) => workId)); if (cases.length !== plannedIds.size || cases.some(({ entry, testName }) => !plannedIds.has(csfTestId(entry.id, testName)))) { @@ -178,11 +193,21 @@ export async function runWdioStoryTests( phase = 'connection'; const started = Date.now(); activeStarted = started; + const diagnostics: WdioCaseDiagnostics = { phases: {} }; + const measure = async (name: keyof WdioCaseDiagnostics['phases'], action: () => T | Promise): Promise => { + const start = performance.now(); + try { + return await action(); + } finally { + diagnostics.phases[name] = (diagnostics.phases[name] ?? 0) + performance.now() - start; + } + }; const result: WdioStoryTestResult = { storyId: entry.id, ...(testName === undefined ? {} : { testName }), status: 'failed', durationMs: 0, + diagnostics, }; activeResult = result; const testId = csfTestId(entry.id, testName); @@ -218,30 +243,62 @@ export async function runWdioStoryTests( const clickMode = resolveDesktopClickMode(descriptor, settings.clickMode, settings.input); const inputMode = settings.input ?? 'compatibility'; // The supervisor owns the session, so even a crashed or timed-out Node worker cannot strand it. - const desktop = await connectDesktopWebdriver({ - clickMode, - inputMode, - launchMode: 'attach', - platformName: options.platform === 'macos' ? 'macos' : 'windows', - targetId: options.targetId, - url: options.url, - }); + const desktop = await measure('sessionAttachMs', () => + connectDesktopWebdriver({ + clickMode, + inputMode, + launchMode: 'attach', + platformName: options.platform === 'macos' ? 'macos' : 'windows', + targetId: options.targetId, + url: options.url, + }), + ); + let activityCursor: WindowActivityCursor | undefined; let failure: unknown; try { phase = 'manifest verification'; - const live = await desktop.listStories(); + const live = await measure('manifestMs', () => desktop.listStories()); if (live.endpoint !== options.platform || live.platformManifestDigest !== options.manifest.platformManifestDigest) { throw new Error('The running Storybook manifest is stale or targets a different platform. Restart storybook driver and the app.'); } // The driver refuses input outside the policy before dispatch, so a lane holds even on a host with no masks. A // driver that does not confirm the policy fails closed rather than running cases it cannot hold or attribute. phase = 'input policy verification'; - const confirmed = desktop.browser.capabilities['furn:inputMode']; - if (confirmed !== inputMode) throw new InputPolicyError(inputMode, confirmed); - const missing = missingDesktopRequirements(descriptor, desktop.browser.capabilities['furn:features'], clickMode); - if (missing.length) throw new MissingRequirementsError(missing); + await measure('policyMs', () => { + const confirmed = desktop.browser.capabilities['furn:inputMode']; + if (confirmed !== inputMode) throw new InputPolicyError(inputMode, confirmed); + const missing = missingDesktopRequirements(descriptor, desktop.browser.capabilities['furn:features'], clickMode); + if (missing.length) throw new MissingRequirementsError(missing); + }); + if (desktop.session.recordsWindowEvents()) { + activityCursor = await measure('activityReadMs', async () => { + try { + return windowActivityCursor(await desktop.session.getWindowEvents()); + } catch { + diagnostics.windowActivity = { status: 'unavailable' }; + return undefined; + } + }); + } else { + diagnostics.windowActivity = { status: 'not-supported' }; + } phase = 'navigation'; - await desktop.openStory(entry.id); + const ready = await measure('navigationMs', () => desktop.openStory(entry.id)); + const observations = ready.diagnostics; + for (const key of ['preparationMs', 'httpMs', 'runtimeReadyMs', 'nativeVerificationMs'] as const) { + const value = observations?.[key]; + if (typeof value === 'number' && Number.isFinite(value) && value >= 0) diagnostics.phases[key] = value; + } + const application = readApplicationIncarnation(observations?.application); + diagnostics.lifecycle = { + contextGeneration: ready.previewGeneration, + mountGeneration: ready.previewGeneration, + jsInitializations: 'not-observed', + ...(application ? { application } : {}), + ...(observations?.nativeMarkerPolls === undefined ? {} : { nativeMarkerPolls: observations.nativeMarkerPolls }), + ...(observations?.authenticatedHellos === undefined ? {} : { authenticatedHellos: observations.authenticatedHellos }), + ...(observations?.bridgeConnections === undefined ? {} : { bridgeConnections: observations.bridgeConnections }), + }; const resultPath = path.join(directory, `${index}.json`); const workerOptions: WdioWorkerOptions = { attachment: { @@ -254,6 +311,7 @@ export async function runWdioStoryTests( platform: options.platform, testName, timeoutMs: settings.timeoutMs, + startedAt: Date.now(), }; const specPath = path.join(directory, `${index}.test.mjs`); fs.writeFileSync( @@ -263,37 +321,46 @@ export async function runWdioStoryTests( { mode: 0o600 }, ); phase = 'execution'; - const outcome = await runWorker( - commands, - { - command: process.execPath, - label: `wdio "${displayName}"`, - // Forced exit races HTTP handle teardown on Windows. The supervisor - // still bounds and terminates workers that do not exit naturally. - args: [ - '--test', - '--test-concurrency=1', - `--test-timeout=${Math.min(settings.timeoutMs + 5000, 2_147_483_647)}`, - `--test-reporter=${settings.reporter}`, - specPath, - ], - cwd: options.config.projectRoot, - env: {}, - category: 'worker', - context, - }, - Math.min(settings.timeoutMs + 5000, 2_147_483_647), - resultPath, - displayName, - options.signal, + const outcome = await measure('workerMs', () => + runWorker( + commands, + { + command: process.execPath, + label: `wdio "${displayName}"`, + // Forced exit races HTTP handle teardown on Windows. The supervisor + // still bounds and terminates workers that do not exit naturally. + args: [ + '--test', + '--test-concurrency=1', + `--test-timeout=${Math.min(settings.timeoutMs + 5000, 2_147_483_647)}`, + `--test-reporter=${settings.reporter}`, + specPath, + ], + cwd: options.config.projectRoot, + env: {}, + category: 'worker', + context, + }, + Math.min(settings.timeoutMs + 5000, 2_147_483_647), + resultPath, + displayName, + options.signal, + (timings) => Object.assign(diagnostics.phases, timings), + ), ); + if (diagnostics.phases.workerStartupMs !== undefined && diagnostics.phases.workerBodyMs !== undefined) { + diagnostics.phases.workerSettlementMs = Math.max( + 0, + diagnostics.phases.workerMs! - diagnostics.phases.workerStartupMs - diagnostics.phases.workerBodyMs, + ); + } result.status = outcome.status; if (outcome.status === 'skipped') { result.skipReason = outcome.skipReason; } // Read before teardown: the receipts belong to the session and end with it. phase = 'input verification'; - result.input = await readDeliveredInput(desktop, descriptor, clickMode, outcome.status === 'passed'); + result.input = await measure('receiptsMs', () => readDeliveredInput(desktop, descriptor, clickMode, outcome.status === 'passed')); if (result.input.undeclared) throw new UndeclaredInputError(describeUndeclaredDesktopInput(result.input.undeclared)); } catch (error) { if (error instanceof MissingRequirementsError) { @@ -332,12 +399,24 @@ export async function runWdioStoryTests( writeDesktopStorybookFailure(`evidence for "${displayName}"`, evidenceError, options.errorOutput); } // What a failed case delivered attributes the failure. A session that cannot answer leaves it unknown. - if (phase === 'execution') result.input = await readDeliveredInput(desktop, descriptor, clickMode, false).catch(() => undefined); + if (phase === 'execution') + result.input = await measure('receiptsMs', () => readDeliveredInput(desktop, descriptor, clickMode, false)).catch( + () => undefined, + ); } } + if (activityCursor) { + await measure('activityReadMs', async () => { + try { + diagnostics.windowActivity = summarizeWindowActivity(await desktop.session.getWindowEvents(), activityCursor); + } catch { + diagnostics.windowActivity = { status: 'unavailable' }; + } + }); + } phase = 'session cleanup'; try { - await desktop.delete(); + await measure('cleanupMs', () => desktop.delete()); } catch (cleanupError) { options.report?.fail(cleanupError, 'cleanup', testId); throw new AggregateError( @@ -417,6 +496,7 @@ export async function runWdioStoryTests( : 'skipped', ...(infrastructureFailure === undefined ? {} : { error: errorMessage(infrastructureFailure) }), tests: results, + diagnostics: { extractionMs, sourceExtractions, totalMs: performance.now() - runStarted }, }); options.report?.emit({ kind: 'artifact', @@ -466,6 +546,7 @@ async function runWorker( resultPath: string, storyId: string, signal?: AbortSignal, + onTimings?: (timings: WdioWorkerTimings) => void, ): Promise { const worker = runner.start(command); let timer: ReturnType | undefined; @@ -489,6 +570,7 @@ async function runWorker( }), ]); const outcome: unknown = fs.existsSync(resultPath) ? JSON.parse(fs.readFileSync(resultPath, 'utf8')) : undefined; + if (outcome && typeof outcome === 'object' && 'timings' in outcome) onTimings?.(readWorkerTimings(outcome.timings)); if ( typeof outcome === 'object' && outcome !== null && diff --git a/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs b/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs index c364a2d321..4ea905b295 100644 --- a/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs +++ b/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs @@ -214,6 +214,23 @@ async function main() { namedTimeout = error.results.map(({ testName, status }) => ({ testName, status })); } const passed = await run('pass'); + assert.ok(passed[0].diagnostics.lifecycle.contextGeneration >= 1); + assert.equal(passed[0].diagnostics.lifecycle.mountGeneration, passed[0].diagnostics.lifecycle.contextGeneration); + assert.equal(passed[0].diagnostics.lifecycle.jsInitializations, 'not-observed'); + for (const phase of [ + 'sessionAttachMs', + 'manifestMs', + 'policyMs', + 'navigationMs', + 'workerStartupMs', + 'workerBodyMs', + 'workerMs', + 'receiptsMs', + 'cleanupMs', + ]) { + assert.equal(typeof passed[0].diagnostics.phases[phase], 'number', `Missing phase ${phase}`); + assert.ok(passed[0].diagnostics.phases[phase] >= 0); + } let skipped; try { await run('skip'); diff --git a/packages/agentic/storybook-desktop/src/testing/worker.ts b/packages/agentic/storybook-desktop/src/testing/worker.ts index 4d9607a12d..a328b08219 100644 --- a/packages/agentic/storybook-desktop/src/testing/worker.ts +++ b/packages/agentic/storybook-desktop/src/testing/worker.ts @@ -17,6 +17,7 @@ export type WdioWorkerOptions = { timeoutMs: number; platform: DesktopEndpoint; testName?: string; + startedAt?: number; }; export function registerWdioStoryTest(options: WdioWorkerOptions, callback: WdioStoryTest): void { @@ -30,13 +31,20 @@ export function registerWdioStoryTest(options: WdioWorkerOptions, callback: Wdio report?.onStop?.(() => controller.abort(new Error('Owned story worker stop requested.'))); let recorded = false; let failure: unknown; + let startupMs: number | undefined; + let bodyStarted: number | undefined; + let bodyMs: number | undefined; + const timings = () => ({ + ...(startupMs === undefined ? {} : { workerStartupMs: startupMs }), + ...(bodyStarted === undefined ? {} : { workerBodyMs: bodyMs ?? performance.now() - bodyStarted }), + }); const recordFailure = (error: unknown) => { failure = error; const diagnostic = report?.reporter.redact(formatDesktopStorybookError(error)) ?? formatDesktopStorybookError(error); try { fs.writeFileSync( options.resultPath, - JSON.stringify({ status: 'failed', error: diagnostic, failure: report?.reporter.failure(error, 'test') }), + JSON.stringify({ status: 'failed', error: diagnostic, failure: report?.reporter.failure(error, 'test'), timings: timings() }), { mode: 0o600 }, ); recorded = true; @@ -59,22 +67,32 @@ export function registerWdioStoryTest(options: WdioWorkerOptions, callback: Wdio } signal.throwIfAborted(); let skipReason: string | undefined; - await callback({ - browser: desktop.browser, - desktop, - expect, - platform: options.platform, - signal, - skip: (reason) => { - if (!reason?.trim()) { - throw new TypeError('Skipping a wdio story requires a reason.'); - } - skipReason = reason; - context.skip(reason); - }, - }); + if (options.startedAt !== undefined) startupMs = Math.max(0, Date.now() - options.startedAt); + bodyStarted = performance.now(); + try { + await callback({ + browser: desktop.browser, + desktop, + expect, + platform: options.platform, + signal, + skip: (reason) => { + if (!reason?.trim()) { + throw new TypeError('Skipping a wdio story requires a reason.'); + } + skipReason = reason; + context.skip(reason); + }, + }); + } finally { + bodyMs = performance.now() - bodyStarted; + } signal.throwIfAborted(); - fs.writeFileSync(options.resultPath, JSON.stringify(skipReason ? { status: 'skipped', skipReason } : { status: 'passed' })); + fs.writeFileSync( + options.resultPath, + JSON.stringify({ ...(skipReason ? { status: 'skipped', skipReason } : { status: 'passed' }), timings: timings() }), + { mode: 0o600 }, + ); recorded = true; } catch (error) { recordFailure(error); From 0aef1090719546fd3eb23c789d357e6de46fe615 Mon Sep 17 00:00:00 2001 From: Jason Morse Date: Thu, 12 Nov 2026 16:39:19 +0000 Subject: [PATCH 2/5] Record macOS activation boundaries and flashing experiment gate Part of #4435. Preserve independent server/provider foreground and hit-test checks while concurrent physical-input corrections are qualified; record the non-activating popup-read proposal without changing native semantics. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../references/current-status.md | 26 +++++++++++++++++++ .../references/remaining-work.md | 21 +++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/packages/agentic/desktop-driver/references/current-status.md b/packages/agentic/desktop-driver/references/current-status.md index 12b95163e2..a44fcb6473 100644 --- a/packages/agentic/desktop-driver/references/current-status.md +++ b/packages/agentic/desktop-driver/references/current-status.md @@ -498,3 +498,29 @@ This timing implementation changes no Swift source, physical-input activation, foreground checks, hit-testing, focus, or window-switch policy. It does not qualify those paths or remove provider-local checks. Windows Fabric and Win32 Paper were **not run locally** for this change. + +### Activation trace and deferred flashing change + +The W3C click path activates before its server-side hit test. The operation +click path conditionally raises an externally obscured target and rechecks; +in-window obstruction remains a no-effect refusal. The native physical-click +provider independently activates under its physical-input scope, rereads the +target, and verifies ownership of the click point. These are separate +postconditions, not interchangeable duplicate calls. Removing the server +activation can move hit-testing before foreground establishment; removing the +provider activation assumes ownership cannot change between commands and +weakens callers that use the provider independently. + +macOS activation sets main/raise attributes even when already frontmost. +The menu test helper `inWindowWith` also calls activating `switchToWindow` +while polling each candidate window and restoring its owner. Both are +plausible flashing contributors, not established visual causes in the +observational ListboxItem cohort. + +No activation optimization is shipped here. Concurrent physical-input +corrections retain foreground/main-window postconditions, add a workspace +fallback when acknowledged AX activation is ineffective, and keep genuine +occlusion refused. An unqualified dedupe must not fork those semantics. +Zero activation counters in this read-only cohort do not meet the optimization +gate. The compatible experiment and its qualification conditions are tracked +in [remaining work](remaining-work.md#macos-activation-and-popup-polling). diff --git a/packages/agentic/desktop-driver/references/remaining-work.md b/packages/agentic/desktop-driver/references/remaining-work.md index ae4bca63c8..fd3442eee4 100644 --- a/packages/agentic/desktop-driver/references/remaining-work.md +++ b/packages/agentic/desktop-driver/references/remaining-work.md @@ -55,6 +55,27 @@ contracts into the owning references and remove completed work here. ## 1. Architecture decision +### macOS activation and popup polling + +Part of #4435. The 2026-11-12 refresh-reduction work deliberately leaves +activation semantics unchanged; the [status ledger](current-status.md) +records the source trace and bounded observational evidence. + +Measure activating menu/popup polling before changing it. Trial explicitly +non-activating, window-scoped reads rather than silently changing +`switchToWindow`. Separately, skipping a raise requires fresh foreground, +main-window, and visibility evidence within the provider's physical-input +scope. Neither recent server activation nor an acknowledged AX request alone +proves that postcondition. Preserve independent provider callers, real +occlusion refusal, hit-testing, focus, cancellation, and input serialization. + +Completion requires paired target-window recordings and matching physical, +accessibility, popup, and multiwindow qualification on both macOS renderers, +compatible with the concurrent physical-input corrections. No implementation +or flashing reduction is claimed by this measurement-only decision. + +### Shared operation architecture + **Build an MCP-first agent interface over a transport-neutral operation layer inside the existing driver service. Keep the CLI as a thin secondary client. Do not create another permanent daemon.** From 7742341ab0dcb94e52f37071d178bbb9642bdd37 Mon Sep 17 00:00:00 2001 From: Jason Morse Date: Thu, 12 Nov 2026 17:26:51 +0000 Subject: [PATCH 3/5] Select driver-owned stories directly and cache extracted test maps per run Part of #4435. Separate dispatch from protected policy, preserve authenticated readiness and per-case remount/session/worker isolation, and recheck source hashes plus manifest digests on cached extractions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .changeset/calm-story-observations.md | 2 + packages/agentic/storybook-desktop/README.md | 5 ++ .../storybook-desktop/config/server.cjs | 1 + .../storybook-desktop/references/readiness.md | 21 ++++- .../references/story-tests.md | 11 ++- .../src/cli/smokeTests.test.ts | 19 +++++ .../storybook-desktop/src/cli/smokeTests.ts | 3 + .../StorybookChannelOrchestrator.test.ts | 85 +++++++++++++++++++ .../driver/StorybookChannelOrchestrator.ts | 42 +++++---- .../src/testing/runWdioTests.ts | 13 ++- .../src/testing/wdioExtractionCache.test.ts | 61 +++++++++++++ .../src/testing/wdioExtractionCache.ts | 19 +++++ .../src/testing/wdioRunner.contract.cjs | 24 ++++++ 13 files changed, 280 insertions(+), 26 deletions(-) create mode 100644 packages/agentic/storybook-desktop/src/testing/wdioExtractionCache.test.ts create mode 100644 packages/agentic/storybook-desktop/src/testing/wdioExtractionCache.ts diff --git a/.changeset/calm-story-observations.md b/.changeset/calm-story-observations.md index a8effbc392..4ae834711a 100644 --- a/.changeset/calm-story-observations.md +++ b/.changeset/calm-story-observations.md @@ -4,3 +4,5 @@ --- Add bounded, service-local story selection timings and per-case execution/lifecycle observations without changing authenticated readiness, preview remounts, or worker/session isolation. Part of #4435. + +Use authenticated direct navigation for driver-owned story selection and cache complete extracted test maps by source content within the owning run. Preserve normal args support, protected args refusal, manifest-digest verification, and the legacy HTTP default for direct library consumers. diff --git a/packages/agentic/storybook-desktop/README.md b/packages/agentic/storybook-desktop/README.md index 0c4f036dd3..5858875674 100644 --- a/packages/agentic/storybook-desktop/README.md +++ b/packages/agentic/storybook-desktop/README.md @@ -26,6 +26,11 @@ live progress. Executable story artifacts include bounded phase/lifecycle observations; see [story-test diagnostics](references/story-tests.md#timing-and-lifecycle-observations). Part of #4435. +Driver-owned navigation selects directly on the authenticated runtime socket, +then verifies correlated readiness and the native marker without a competing +HTTP mutation; see [readiness](references/readiness.md). Test extraction is +cached by source content within its owning run, without reusing previews, +workers, or sessions. Agent-served channels reject alternate HTTP mutations and never relay client WebSocket frames; selection/reset go through the session-owning driver grant, and args remain unsupported. Non-agent channels are explicitly legacy diff --git a/packages/agentic/storybook-desktop/config/server.cjs b/packages/agentic/storybook-desktop/config/server.cjs index c8cfbb5403..1d7eca09b9 100644 --- a/packages/agentic/storybook-desktop/config/server.cjs +++ b/packages/agentic/storybook-desktop/config/server.cjs @@ -273,6 +273,7 @@ function createChannelOrchestrator({ channelServer, driverManifest, serverUrl, S serverUrl, recovery, controlled: driverManifest.agent !== undefined, + selectionTransport: 'direct', }); } diff --git a/packages/agentic/storybook-desktop/references/readiness.md b/packages/agentic/storybook-desktop/references/readiness.md index e454b5f262..70100a6d4a 100644 --- a/packages/agentic/storybook-desktop/references/readiness.md +++ b/packages/agentic/storybook-desktop/references/readiness.md @@ -42,10 +42,23 @@ replacement awaits its initial render before it becomes selectable. The local provider context carries the session lifetime ID and abort signal separately from serializable inputs. No authenticated bridge means prompt `bridge-unavailable`, with endpoint/renderer and effect `none`, before dispatch. -Controlled selection sends prepare and navigation directly to the authenticated -socket, waits for correlated runtime readiness, then the driver verifies the -native marker within the same render deadline. Legacy unprotected selection -additionally waits for upstream HTTP navigation. Disconnect promptly rejects +Driver-owned normal and controlled selection send prepare and navigation +directly to the authenticated socket, wait for correlated runtime readiness, +then verify the native marker within the same render deadline. No competing +HTTP mutation is sent. `selectionTransport` chooses dispatch independently of +the `controlled` protection policy. Direct library consumers retain the +`legacy-http` default unless they explicitly choose `direct`; legacy HTTP +selection additionally waits for upstream navigation. A controlled service +always forces direct selection, regardless of that option, and still refuses +args. Normal direct selection continues to support args updates. + +The driver does not broadcast its direct selection mutation to other channel +clients: only the authenticated runtime receives it. Existing unprotected +runtime-event relaying is unchanged. This does not protect the legacy +channel's otherwise-unprotected HTTP, WebSocket relay, or MCP endpoints; +agent services still require the separate controlled adapter. + +Disconnect promptly rejects pending work and aborts any legacy HTTP request. Cancellation, disconnect, or deadline after dispatch fences that socket and leaves the effect unknown: the driver retains target quarantine. Late ready/error events diff --git a/packages/agentic/storybook-desktop/references/story-tests.md b/packages/agentic/storybook-desktop/references/story-tests.md index 20df61f1a1..de9babdff2 100644 --- a/packages/agentic/storybook-desktop/references/story-tests.md +++ b/packages/agentic/storybook-desktop/references/story-tests.md @@ -157,6 +157,13 @@ per-case timeout applies; assertion failure does not prevent independent cases from running, while connection or cleanup failures stop execution. No matching case is an error. Restart the driver and reload the app after editing callbacks because their digest participates in manifest freshness. +Extraction maps are cached by complete source-content hash and source location +within the owning test run. Smoke shares that cache across its per-story runner +calls. Every use rereads/hashes the source and still compares the selected +export's digest with the manifest; same-size or timestamp-preserving edits do +not bypass freshness or descriptor validation. The cache keeps only the latest +map per file and never survives its owning run. It caches no session, worker, +readiness result, or native context. `test --list` discovers IDs without executing callbacks; `--test ` filters names. `test` operates against an already running app/driver; smoke owns the complete app lifecycle. @@ -165,7 +172,9 @@ owns the complete app lifecycle. Part of #4435. The linked **Executable story results** `run.json` artifact records `diagnostics` without changing test selection, readiness, or isolation. -Run-level fields are `extractionMs`, `sourceExtractions`, and `totalMs`. +Run-level fields are `extractionMs`, `sourceExtractions`, `sourceCacheHits`, and +`totalMs`. Extraction counts are full-file parses; cache hits still include +source rereading/hashing and manifest-digest verification. Each attempted case records `diagnostics.phases`: - `sessionAttachMs`, `manifestMs`, and `policyMs`; diff --git a/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts b/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts index d18d37bc8d..f95d1d2f8b 100644 --- a/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts +++ b/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts @@ -194,6 +194,25 @@ describe('runDesktopStorybookSmokeTests', () => { expect(deleteSession).toHaveBeenCalledTimes(1); }); + test('shares one extraction cache across per-story calls, but not across separate smoke runs', async () => { + options.manifest = { + ...manifest, + entries: [buttonId, inputId].map((id) => ({ ...story(id, true), tests: undefined })), + }; + const connect = jest.fn(); + const runWdio: jest.MockedFunction = jest.fn(async ({ story }) => [ + { storyId: story!, status: 'passed' as const, durationMs: 1 }, + ]); + await runDesktopStorybookSmokeTests(options, connect, runWdio); + const first = runWdio.mock.calls[0][0].extractionCache; + expect(first).toBeDefined(); + expect(runWdio.mock.calls[1][0].extractionCache).toBe(first); + await runDesktopStorybookSmokeTests(options, connect, runWdio); + expect(runWdio.mock.calls[2][0].extractionCache).not.toBe(first); + expect(runWdio.mock.calls[3][0].extractionCache).toBe(runWdio.mock.calls[2][0].extractionCache); + expect(connect).not.toHaveBeenCalled(); + }); + test.each(['storyPattern', 'testTag'] as const)('rejects an empty %s before connecting', async (key) => { options[key] = ' '; const connect = jest.fn(); diff --git a/packages/agentic/storybook-desktop/src/cli/smokeTests.ts b/packages/agentic/storybook-desktop/src/cli/smokeTests.ts index 245b4f603f..f23f05fd1e 100644 --- a/packages/agentic/storybook-desktop/src/cli/smokeTests.ts +++ b/packages/agentic/storybook-desktop/src/cli/smokeTests.ts @@ -14,6 +14,7 @@ import type { Platforms } from '../config/platforms.js'; import { resolveWdioOptions } from '../config/wdio.js'; import { describeWdioCounts, hasExecutedWdioCase } from '../testing/laneVerdict.js'; import { runWdioStoryTests, WdioStoryTestRunError, type WdioStoryTestResult } from '../testing/runWdioTests.js'; +import { WdioExtractionCache } from '../testing/wdioExtractionCache.js'; import { decideWdioSelection, describeWdioExclusions, plannedWdioWork, type WdioSelectionContext } from '../testing/selection.js'; import { writeDesktopStorybookFailure, type DesktopStorybookErrorOutput } from '../../config/diagnostics.cjs'; import type { DesktopCommandRunner } from './commandRunner.js'; @@ -97,6 +98,7 @@ async function runSmokeTests( const startedAt = new Date().toISOString(); const tests: DesktopStoryTestResult[] = []; const wdio: WdioStoryTestResult[] = []; + const extractionCache = new WdioExtractionCache(); const failures: unknown[] = []; const staticFailures = new Map(); for (const storyId of storyIds) options.report?.emit({ kind: 'work-selected', payload: { workId: `smoke:${storyId}`, label: storyId } }); @@ -142,6 +144,7 @@ async function runSmokeTests( signal: options.signal, optional: true, announced: true, + extractionCache, }, options.commandRunner, )), diff --git a/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.test.ts b/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.test.ts index b4aefaaa3e..b8d1381722 100644 --- a/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.test.ts +++ b/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.test.ts @@ -102,6 +102,7 @@ function setup( recovery?: BridgeEpochRecovery, renderer: 'fabric' | 'paper' = 'fabric', controlled = false, + selectionTransport: 'direct' | 'legacy-http' = 'legacy-http', ) { const server = Object.assign(new EventEmitter(), { clients: new Set() }); const client = new Client(); @@ -116,10 +117,94 @@ function setup( timeoutMs, recovery, controlled, + selectionTransport, }); return { client, errorOutput, fetch, orchestrator, server }; } +describe('driver-owned direct selection', () => { + test.each(['fabric', 'paper'] as const)('%s uses only the authenticated socket and keeps normal args support', async (renderer) => { + const { client, orchestrator, server, fetch } = setup(undefined, 100, undefined, renderer, false, 'direct'); + const foreign = new Client(); + server.emit('connection', foreign); + client.authenticate(); + const diagnostics = {}; + const selection = orchestrator.selectStory(request, { + lifetimeId: 'owner', + signal: new AbortController().signal, + diagnostics, + }); + foreign.ready({ connectionEpoch: client.epoch }); + client.ready({ requestId: 'old-request' }); + client.ready({ connectionEpoch: 'old-epoch' }); + client.receive('furn:desktop:story-error', { ...request, connectionEpoch: 'old-epoch' }); + await expect(orchestrator.getCurrentStory()).resolves.toBeNull(); + client.ready(); + await expect(selection).resolves.toMatchObject({ ...request, connectionEpoch: client.epoch, previewGeneration: 1 }); + expect(client.sent.slice(-2).map(({ type }) => type)).toEqual(['furn:desktop:prepare-story', 'setCurrentStory']); + expect(diagnostics).not.toHaveProperty('httpMs'); + const reset = orchestrator.resetStory({ ...request, requestId: 'reset' }); + client.ready({ requestId: 'reset', previewGeneration: 2 }); + await expect(reset).resolves.toMatchObject({ requestId: 'reset', previewGeneration: 2 }); + await orchestrator.updateArgs(request.storyId, { label: 'normal args remain supported' }); + expect(client.sent.at(-1)).toMatchObject({ type: 'updateStoryArgs' }); + expect(foreign.sent).toHaveLength(1); + expect(fetch).not.toHaveBeenCalled(); + }); + + test('post-dispatch cancellation quarantines direct selection despite delayed readiness', async () => { + const { client, orchestrator, fetch } = setup(undefined, 100, undefined, 'fabric', false, 'direct'); + client.authenticate(); + const controller = new AbortController(); + const selection = orchestrator.selectStory(request, { lifetimeId: 'owner', signal: controller.signal }); + controller.abort(); + await expect(selection).rejects.toHaveProperty('effect', 'unknown'); + client.ready(); + await expect(orchestrator.resetStory({ ...request, requestId: 'next' })).rejects.toMatchObject({ + data: { reason: 'relaunch-required' }, + effect: 'none', + }); + expect(fetch).not.toHaveBeenCalled(); + }); + + test.each(['before-dispatch', 'disconnect', 'render-error', 'deadline'] as const)( + 'direct selection preserves failure effects for %s', + async (failure) => { + const { client, orchestrator, fetch } = setup(undefined, 10, undefined, 'fabric', false, 'direct'); + client.authenticate(); + const controller = new AbortController(); + if (failure === 'before-dispatch') controller.abort(); + const selection = orchestrator.selectStory(request, { lifetimeId: 'owner', signal: controller.signal }); + if (failure === 'disconnect') { + client.readyState = 3; + client.emit('close'); + } else if (failure === 'render-error') { + client.receive('furn:desktop:story-error', { ...request, connectionEpoch: client.epoch }); + } + await expect(selection).rejects.toHaveProperty('effect', failure === 'before-dispatch' ? 'none' : 'unknown'); + await expect(orchestrator.getCurrentStory()).resolves.toBeNull(); + if (failure !== 'before-dispatch') { + await expect(orchestrator.selectStory({ ...request, requestId: 'next' })).rejects.toMatchObject({ + data: { reason: 'relaunch-required' }, + effect: 'none', + }); + } + expect(fetch).not.toHaveBeenCalled(); + }, + ); + + test.each(['direct', 'legacy-http'] as const)('transport %s cannot weaken controlled args refusal or request HTTP', async (transport) => { + const { client, orchestrator, fetch } = setup(undefined, 100, undefined, 'fabric', true, transport); + client.authenticate(); + const selection = orchestrator.selectStory(request); + client.ready(); + await selection; + await expect(orchestrator.updateArgs(request.storyId, { private: 'do-not-relay' })).rejects.toHaveProperty('effect', 'none'); + expect(JSON.stringify(client.sent)).not.toContain('do-not-relay'); + expect(fetch).not.toHaveBeenCalled(); + }); +}); + describe('controlled channel endpoint boundary', () => { test('HTTP and foreign WebSocket selection, prepare and args never reach the authenticated runtime', async () => { const errors = jest.fn(); diff --git a/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.ts b/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.ts index e20fb1bc8c..7bb29f3ca7 100644 --- a/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.ts +++ b/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.ts @@ -34,8 +34,10 @@ export type StorybookChannelOrchestratorOptions = { timeoutMs?: number; errorOutput?: DesktopStorybookErrorOutput; recovery?: BridgeEpochRecovery; - /** Exclusive transport: dispatch only to the authenticated runtime, never upstream HTTP. */ + /** Protected policy: refuse args and require authenticated direct selection. */ controlled?: boolean; + /** Dispatch choice only. Legacy HTTP remains the default for direct library consumers. */ + selectionTransport?: 'direct' | 'legacy-http'; }; type PendingSelection = { @@ -52,6 +54,7 @@ export class StorybookChannelOrchestrator implements StoryOrchestrator { private readonly timeoutMs: number; private readonly errorOutput: DesktopStorybookErrorOutput; private readonly controlled: boolean; + private readonly selectionTransport: 'direct' | 'legacy-http'; private readonly pending = new Map(); private readonly epochs = new WeakMap(); private readonly usedRequests = new WeakMap>(); @@ -78,6 +81,7 @@ export class StorybookChannelOrchestrator implements StoryOrchestrator { errorOutput = process.stderr, recovery, controlled = driverManifest.agent !== undefined, + selectionTransport = 'legacy-http', }: StorybookChannelOrchestratorOptions) { this.driverManifest = driverManifest; this.fetch = fetch; @@ -85,6 +89,7 @@ export class StorybookChannelOrchestrator implements StoryOrchestrator { this.timeoutMs = timeoutMs; this.errorOutput = errorOutput; this.controlled = driverManifest.agent !== undefined || controlled; + this.selectionTransport = this.controlled ? 'direct' : selectionTransport; this.journal = recovery ? new BridgeEpochJournal(recovery, { endpoint: driverManifest.endpoint, @@ -163,23 +168,26 @@ export class StorybookChannelOrchestrator implements StoryOrchestrator { context.diagnostics.bridgeConnections = this.bridgeConnections; } const httpStarted = performance.now(); - const navigation = this.controlled - ? Promise.resolve(client.send(JSON.stringify({ type: 'setCurrentStory', args: [{ storyId: request.storyId, viewMode: 'story' }] }))) - : this.fetch(`${this.serverUrl}/select-story-sync/${encodeURIComponent(request.storyId)}`, { - method: 'POST', - signal: controller.signal, - }) - .then(async (response) => { - if (!response.ok && response.status !== 408) { - const detail = await response.text(); - throw new Error( - `Storybook failed to select "${request.storyId}" with status ${response.status}${detail ? `: ${detail}` : '.'}`, - ); - } + const navigation = + this.selectionTransport === 'direct' + ? Promise.resolve( + client.send(JSON.stringify({ type: 'setCurrentStory', args: [{ storyId: request.storyId, viewMode: 'story' }] })), + ) + : this.fetch(`${this.serverUrl}/select-story-sync/${encodeURIComponent(request.storyId)}`, { + method: 'POST', + signal: controller.signal, }) - .finally(() => { - if (context?.diagnostics) context.diagnostics.httpMs = performance.now() - httpStarted; - }); + .then(async (response) => { + if (!response.ok && response.status !== 408) { + const detail = await response.text(); + throw new Error( + `Storybook failed to select "${request.storyId}" with status ${response.status}${detail ? `: ${detail}` : '.'}`, + ); + } + }) + .finally(() => { + if (context?.diagnostics) context.diagnostics.httpMs = performance.now() - httpStarted; + }); const [, result] = await Promise.race([Promise.all([navigation, ready]), failure]); this.currentStory = result; return result; diff --git a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts index 2145228d94..8746a7aed1 100644 --- a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts +++ b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts @@ -20,7 +20,7 @@ import type { DesktopStorybookConfig } from '../config/makeDesktopStorybookConfi import type { Platforms } from '../config/platforms.js'; import { resolveWdioOptions, type DesktopStorybookWdioOptions } from '../config/wdio.js'; import { NodeDesktopCommandRunner, type DesktopCommandRunner, type PreparedDesktopCommand } from '../cli/commandRunner.js'; -import { extractWdioTests } from './extractWdioTests.js'; +import { WdioExtractionCache } from './wdioExtractionCache.js'; import { assessWdioCohort } from './laneVerdict.js'; import { readApplicationIncarnation, @@ -72,6 +72,7 @@ export type RunWdioStoryTestsOptions = DesktopStorybookWdioOptions & { optional?: boolean; /** The caller already reported the selection and announced the planned work, so this call must not repeat it. */ announced?: boolean; + extractionCache?: WdioExtractionCache; }; export class WdioStoryTestRunError extends Error { @@ -137,11 +138,15 @@ export async function runWdioStoryTests( throw new Error(`No executable wdio story tests matched the requested selection${describeWdioExclusions(selection)}.`); } const extractionStarted = performance.now(); + const extractionCache = options.extractionCache ?? new WdioExtractionCache(); let sourceExtractions = 0; + let sourceCacheHits = 0; const cases = selection.stories.flatMap((entry) => { const sourceFile = path.join(options.config.resolvePackage(entry.packageName).root, entry.sourcePath); - sourceExtractions++; - const extracted = extractWdioTests(fs.readFileSync(sourceFile, 'utf8'), sourceFile).get(entry.wdio!.exportName); + const { tests, cacheHit } = extractionCache.read(sourceFile); + if (cacheHit) sourceCacheHits++; + else sourceExtractions++; + const extracted = tests.get(entry.wdio!.exportName); if (!extracted || extracted.digest !== entry.wdio!.digest) { throw new Error(`Executable test source changed for ${entry.id}; regenerate the Storybook manifest.`); } @@ -496,7 +501,7 @@ export async function runWdioStoryTests( : 'skipped', ...(infrastructureFailure === undefined ? {} : { error: errorMessage(infrastructureFailure) }), tests: results, - diagnostics: { extractionMs, sourceExtractions, totalMs: performance.now() - runStarted }, + diagnostics: { extractionMs, sourceExtractions, sourceCacheHits, totalMs: performance.now() - runStarted }, }); options.report?.emit({ kind: 'artifact', diff --git a/packages/agentic/storybook-desktop/src/testing/wdioExtractionCache.test.ts b/packages/agentic/storybook-desktop/src/testing/wdioExtractionCache.test.ts new file mode 100644 index 0000000000..6a2be0b079 --- /dev/null +++ b/packages/agentic/storybook-desktop/src/testing/wdioExtractionCache.test.ts @@ -0,0 +1,61 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { WdioExtractionCache } from './wdioExtractionCache.js'; + +let root: string; +let sourceFile: string; + +beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'storybook-extraction-cache-')); + sourceFile = path.join(root, 'fixture.stories.ts'); +}); + +afterEach(() => fs.rmSync(root, { force: true, recursive: true })); + +test('shares the complete export map within its owner, never across independently owned runs', () => { + fs.writeFileSync(sourceFile, `export const First = { wdio: () => 'first' }; export const Second = { wdio: () => 'second' };`); + const cache = new WdioExtractionCache(); + const initial = cache.read(sourceFile); + expect([...initial.tests.keys()]).toEqual(['First', 'Second']); + const reused = cache.read(sourceFile); + expect(reused.cacheHit).toBe(true); + expect(reused.tests.get('Second')).toEqual(initial.tests.get('Second')); + expect(new WdioExtractionCache().read(sourceFile).cacheHit).toBe(false); +}); + +test('rechecks source content despite unchanged size and mtime and rejects newly invalid descriptors', () => { + const cache = new WdioExtractionCache(); + fs.writeFileSync(sourceFile, `export const First = { wdio: () => 'first' };`); + fs.utimesSync(sourceFile, 1_700_000_000, 1_700_000_000); + const initial = cache.read(sourceFile); + const stat = fs.statSync(sourceFile); + fs.writeFileSync(sourceFile, `export const First = { wdio: () => 'other' };`); + fs.utimesSync(sourceFile, stat.atime, stat.mtime); + expect(fs.statSync(sourceFile).mtimeMs).toBe(stat.mtimeMs); + const edited = cache.read(sourceFile); + expect(edited.cacheHit).toBe(false); + expect(edited.tests.get('First')!.digest).not.toBe(initial.tests.get('First')!.digest); + fs.writeFileSync( + sourceFile, + `import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; + export const First = { wdio: [desktopIt('case', { input: 'invalid' }, () => {})] };`, + ); + expect(() => cache.read(sourceFile)).toThrow(/input/); +}); + +test('identical content in different source locations retains location-specific helper resolution', () => { + fs.mkdirSync(path.join(root, 'other')); + const otherSource = path.join(root, 'other', 'fixture.stories.ts'); + const source = `export const First = { wdio: async () => { await import('./helper.ts'); } };`; + fs.writeFileSync(path.join(root, 'helper.ts'), 'export const value = 1;'); + fs.writeFileSync(path.join(root, 'other', 'helper.ts'), 'export const value = 2;'); + fs.writeFileSync(sourceFile, source); + fs.writeFileSync(otherSource, source); + const cache = new WdioExtractionCache(); + const first = cache.read(sourceFile); + const other = cache.read(otherSource); + expect(other.cacheHit).toBe(false); + expect(other.tests.get('First')!.tests[0].code).not.toBe(first.tests.get('First')!.tests[0].code); +}); diff --git a/packages/agentic/storybook-desktop/src/testing/wdioExtractionCache.ts b/packages/agentic/storybook-desktop/src/testing/wdioExtractionCache.ts new file mode 100644 index 0000000000..743d3a5616 --- /dev/null +++ b/packages/agentic/storybook-desktop/src/testing/wdioExtractionCache.ts @@ -0,0 +1,19 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; + +import { extractWdioTests, type ExtractedWdioStory } from './extractWdioTests.js'; + +/** Owned by one test/smoke run, with only the latest source identity retained for each file. */ +export class WdioExtractionCache { + private readonly sources = new Map }>(); + + read(sourceFile: string): { tests: ReadonlyMap; cacheHit: boolean } { + const source = fs.readFileSync(sourceFile, 'utf8'); + const digest = createHash('sha256').update(source).digest('hex'); + const cached = this.sources.get(sourceFile); + if (cached?.digest === digest) return { tests: cached.tests, cacheHit: true }; + const tests = extractWdioTests(source, sourceFile); + this.sources.set(sourceFile, { digest, tests }); + return { tests, cacheHit: false }; + } +} diff --git a/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs b/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs index 4ea905b295..7388449cc4 100644 --- a/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs +++ b/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs @@ -17,12 +17,14 @@ async function main() { { createDesktopDriverStoryHarness, createFakeStoryWindows }, { makeDesktopStorybookConfig }, { DesktopRunReporter }, + { WdioExtractionCache }, ] = await Promise.all([ import(pathToFileURL(path.join(packageRoot, 'lib/driver/storyManifest.js')).href), import(pathToFileURL(path.join(packageRoot, 'lib/testing/runWdioTests.js')).href), import('@fluentui-react-native/desktop-driver/testing'), import(pathToFileURL(path.join(packageRoot, 'lib/config/index.js')).href), import('@fluentui-react-native/desktop-test/reporting'), + import(pathToFileURL(path.join(packageRoot, 'lib/testing/wdioExtractionCache.js')).href), ]); fs.writeFileSync( path.join(root, 'fixture.stories.ts'), @@ -201,6 +203,28 @@ async function main() { } const filtered = await run('named', { test: '*fresh' }); if (filtered.length !== 1 || filtered[0].testName !== 'starts fresh') throw new Error('Named test filtering failed.'); + const extractionCache = new WdioExtractionCache(); + await run('named', { test: '*fresh', extractionCache }); + const extractedReport = JSON.parse(fs.readFileSync(path.join(root, 'artifacts/windows/wdio/run.json'), 'utf8')); + assert.equal(extractedReport.diagnostics.sourceExtractions, 1); + await run('pass', { extractionCache }); + const reusedReport = JSON.parse(fs.readFileSync(path.join(root, 'artifacts/windows/wdio/run.json'), 'utf8')); + assert.equal(reusedReport.diagnostics.sourceExtractions, 0); + assert.equal(reusedReport.diagnostics.sourceCacheHits, 1); + const selectionsBeforeMismatch = selections.length; + await assert.rejects( + run('pass', { + extractionCache, + manifest: { + ...manifest, + entries: manifest.entries.map((entry) => + entry.id === 'fixture-button--pass' ? { ...entry, wdio: { ...entry.wdio, digest: 'stale' } } : entry, + ), + }, + }), + /Executable test source changed/, + ); + assert.equal(selections.length, selectionsBeforeMismatch); let namedTimeout; try { await run('named-timeout'); From 0eac3c6a1f30d1e297175e91fd4294f8346866b8 Mon Sep 17 00:00:00 2001 From: Jason Morse Date: Thu, 12 Nov 2026 18:07:31 +0000 Subject: [PATCH 4/5] Record paired desktop navigation measurements and qualification Part of #4435. Record eight-run baseline/candidate measurements for both macOS renderers, sparse accessible/physical qualification, native contracts, recording limits, deferred gates, and final aggregate results without claiming a flashing or reliability improvement. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../references/current-status.md | 49 ++- .../references/current-status.md | 310 ++++++++++++++++-- 2 files changed, 332 insertions(+), 27 deletions(-) diff --git a/packages/agentic/desktop-driver/references/current-status.md b/packages/agentic/desktop-driver/references/current-status.md index a44fcb6473..b2d0a1d5a8 100644 --- a/packages/agentic/desktop-driver/references/current-status.md +++ b/packages/agentic/desktop-driver/references/current-status.md @@ -482,14 +482,17 @@ authority to clear quarantine. Request/run/epoch matching and fresh native marker verification remain unchanged; provider payloads and private values are not copied into these observations. -The instrumented baseline ran eight independently launched macOS Fabric -accessible cohorts, each with three passing ListboxItem callbacks and one +The instrumented baseline ran eight independently launched accessible cohorts +per macOS renderer, each with three passing ListboxItem callbacks and one declared macOS-unsupported callback. Each passing callback selected a fresh keyed preview/context generation; every cohort retained one verified app incarnation and one authenticated bridge connection during its callbacks. JS initialization/HMR was not observed independently and cannot be inferred -from hello counts. No activation or main/focused-window change was observed in -this observational cohort; a zero advisory count is not visual qualification. +from hello counts. Fabric accepted one hello and Paper two on the same socket. +No activation command or main/focused-window change was observed in these +baseline cohorts; Paper recorded one application deactivation, and 16 of 24 +event samples per renderer carried truncation warnings. A zero advisory count +is not visual qualification. The Storybook evidence ledger, `packages/agentic/storybook-desktop/references/current-status.md`, records selection, timing, variability, and limitations. @@ -499,6 +502,44 @@ activation, foreground checks, hit-testing, focus, or window-switch policy. It does not qualify those paths or remove provider-local checks. Windows Fabric and Win32 Paper were **not run locally** for this change. +The direct-selection Fabric candidate preserved identical coverage, fresh +per-case generations, one app incarnation/connection per lifecycle, and no +within-lifecycle app replacements across eight paired-by-index repetitions. +Runner p50/p95 changed from 7939.07/8845.37 to 6680.04/6995.31 ms; navigation +p50 changed from 753.13 to 274.20 ms. Paper's equivalent eight-plus-eight +comparison changed runner p50/p95 from 7915.99/8755.09 to 7218.78/7811.59 ms +and navigation p50 from 756.77 to 340.59 ms, with identical coverage and +lifecycle invariants. Each side/renderer passed 24 callbacks plus eight +declared unsupported. Sequential before/after blocks, cold startup +differences, and the small observational cohort limit attribution; +native-verification p95 worsened on both renderers (Fabric 168.00→546.11 ms; +Paper 170.07→582.90 ms). There is no universal tail-latency, reliability, +startup, or flashing claim. + +Window-scoped synchronized recordings on both sides/renderers contained four +encoded frames each, followed by held pixels. The unvalidated whole-content +uniformity detector returned zero candidates; persistent chrome and +window-only capture prevent interpreting this as zero blank-preview or +activation-flash episodes. No flash reduction was established. + +Focused candidate qualification on both macOS renderers passed the complete +223-story render traversal plus five accessible callbacks (228 passes per +renderer), and a separate sparse physical selection with 15 callbacks +(238 passes per renderer). Button, Dropdown, native Callout, and modern +Callout paths retained fresh per-case contexts/workers and passed popup +isolation/dismiss/reopen checks. These are not full authored-input suites, +MCP descriptor qualification, native cancellation/fault injection, or +qualification of another branch's physical-input corrections. + +The unchanged standalone macOS helper passed a release build with +`-Xswiftc -warnings-as-errors` and all 448 self-test checks. Focused +`FURN_NATIVE_DRIVER_TEST=1` contracts passed 87 tests, with 16 skipped +(Windows contracts not run locally). The final aggregate passed in the +repository's serial-Jest CI mode after isolating Git discovery for +project-local temporary fixtures and rerunning two timing-sensitive +parallel-test failures. The Storybook evidence ledger records exact commands, +failed attempts, package counts, and the final passed result. + ### Activation trace and deferred flashing change The W3C click path activates before its server-side hit test. The operation diff --git a/packages/agentic/storybook-desktop/references/current-status.md b/packages/agentic/storybook-desktop/references/current-status.md index 7c1d2956b3..595880e12d 100644 --- a/packages/agentic/storybook-desktop/references/current-status.md +++ b/packages/agentic/storybook-desktop/references/current-status.md @@ -16,9 +16,9 @@ parent of the navigation optimization, stacked on PR #4490 ### Method and coverage -The public `DesktopStorybookCli` API ran eight independently launched macOS -Fabric lifecycles in the accessible lane. An injected fetch narrowed only the -CLI's traversal index to the real +The public `DesktopStorybookCli` API ran eight independently launched +lifecycles per macOS renderer in the accessible lane. An injected fetch +narrowed only the CLI's traversal index to the real `components-listboxitem--native-selection` entry; the server's authenticated manifest, runtime, native helper, and authored callbacks were unchanged. Every lifecycle traversed that story, ran its three supported callbacks in @@ -32,22 +32,22 @@ uses inclusive linear interpolation. Phase samples are 24 executed callbacks; runner-total samples are eight lifecycles. Navigation subphases overlap and must not be added together. The first lifecycle can include cold caches. -| Fabric baseline phase | p50 ms | p95 ms | -| ---------------------------- | ------: | ------: | -| Session attach | 1192.22 | 1612.56 | -| Manifest | 2.62 | 3.37 | -| Input policy | 0.01 | 0.04 | -| Preparation | 0.13 | 0.17 | -| Upstream HTTP | 503.23 | 504.56 | -| Correlated runtime readiness | 28.77 | 562.62 | -| Native marker verification | 75.07 | 168.00 | -| Complete navigation | 753.13 | 812.97 | -| Worker startup | 203.00 | 215.85 | -| Worker body | 505.18 | 533.74 | -| Settled worker | 723.34 | 750.74 | -| Receipts | 2.62 | 2.93 | -| Session cleanup | 115.24 | 126.77 | -| Diagnostic native reads | 13.33 | 18.08 | +| Baseline phase | Fabric p50 ms | Fabric p95 ms | Paper p50 ms | Paper p95 ms | +| ---------------------------- | ------------: | ------------: | -----------: | -----------: | +| Session attach | 1192.22 | 1612.56 | 1094.93 | 1609.26 | +| Manifest | 2.62 | 3.37 | 2.60 | 3.14 | +| Input policy | 0.01 | 0.04 | 0.01 | 0.04 | +| Preparation | 0.13 | 0.17 | 0.13 | 0.24 | +| Upstream HTTP | 503.23 | 504.56 | 503.08 | 503.95 | +| Correlated runtime readiness | 28.77 | 562.62 | 25.86 | 458.94 | +| Native marker verification | 75.07 | 168.00 | 97.40 | 170.07 | +| Complete navigation | 753.13 | 812.97 | 756.77 | 792.62 | +| Worker startup | 203.00 | 215.85 | 196.50 | 207.40 | +| Worker body | 505.18 | 533.74 | 523.32 | 556.19 | +| Settled worker | 723.34 | 750.74 | 736.00 | 789.32 | +| Receipts | 2.62 | 2.93 | 2.46 | 2.76 | +| Session cleanup | 115.24 | 126.77 | 108.93 | 115.69 | +| Diagnostic native reads | 13.33 | 18.08 | 6.79 | 9.11 | Runner-total p50/p95 were **7939.07 / 8845.37 ms**; aggregate duration was **63950.71 ms**. Sample variance was **306828.83 ms²** (range @@ -55,10 +55,21 @@ Runner-total p50/p95 were **7939.07 / 8845.37 ms**; aggregate duration was traversal, were 19887.17 / 28433.88 ms, with 171994.83 ms aggregate and 19950017.41 ms² sample variance. These different totals are not interchangeable. +Paper runner-total p50/p95 were **7915.99 / 8755.09 ms**, aggregate +**60909.13 ms**, sample variance **2199477.39 ms²**, and range +4123.59–8794.34 ms. Complete lifecycle p50/p95 were 19928.62 / 62342.92 ms, +aggregate 224024.71 ms, and variance 529506165.46 ms². The first Paper +lifecycle totaled 84945.44 ms including cold startup, but had an unusually +short runner +(4123.59 ms); both remain included. The first Fabric baseline also had +recording/cold-cache overhead. No startup or reliability improvement is +attributed to those differences. + ### Lifecycle and observation limits -Each lifecycle had one verified PID/start-time pair, one authenticated bridge -connection, and one accepted hello. Each callback selected a fresh generation +Each lifecycle had one verified PID/start-time pair and one authenticated bridge +connection. Fabric accepted one hello per baseline lifecycle; Paper accepted +two on the same connection. Each callback selected a fresh generation (2, 3, 4 after the traversal's first generation), with matching keyed-mount and context generations. No within-lifecycle application replacement or new bridge connection was observed. JS initialization/HMR is explicitly **not observed**: @@ -66,8 +77,10 @@ neither a hello nor the absence of a new socket proves a JS initialization count The keyed generation is not an instrumented React effect-mount counter. Native activity reports retained fixed counters only. No activation command, -provider activation, application activation/deactivation, or main/focused-window -change was observed in this observational cohort. `AXUIElementDestroyed` +provider activation, application activation, or main/focused-window change +was observed in these baseline cohorts; Paper recorded one application +deactivation. Sixteen of 24 callback event samples per renderer carried a +truncation warning. `AXUIElementDestroyed` notifications can describe preview elements, not necessarily closed windows. The bounded native event ring can be truncated, and its observations are advisory. These counters do not establish whether a user saw a flash. @@ -85,3 +98,254 @@ sharing, worker-isolation flag, preview reuse, or runtime protocol expansion is implemented by the timing work. Such a trial still needs independently paired benefit and ownership/settlement evidence. Windows Fabric and Win32 Paper: **not run locally**. + +## 2026-11-12: direct-selection candidate + +Part of #4435. Candidate implementation: `06446c276`; instrumentation: +`f0feebc90`. The driver-owned channel factory now uses authenticated direct +selection without an upstream HTTP mutation. Controlled policy remains +independent and still refuses protected args and foreign mutation relaying. +Full extracted export maps are cached by source content/location within one +owning run, including across smoke's per-story calls; manifest digest checks, +remounts, fresh sessions, fresh workers, and native marker verification remain. + +### Fabric before/after + +Eight baseline and eight candidate lifecycles selected identical callbacks in +identical order: 24 passed and eight declared unsupported on each side. +These are sequential before/after blocks, not randomized alternating runs. +Phase samples are 24 callbacks; extraction samples and totals are eight runs. +Values below are p50 / p95 milliseconds. + +| Phase | Baseline | Candidate | +| ---------------------------- | ----------------- | ----------------- | +| Session attach | 1192.22 / 1612.56 | 1088.67 / 1611.77 | +| Manifest | 2.62 / 3.37 | 2.90 / 3.88 | +| Input policy | 0.01 / 0.04 | 0.01 / 0.02 | +| Preparation | 0.13 / 0.17 | 0.15 / 0.20 | +| Upstream HTTP | 503.23 / 504.56 | Not dispatched | +| Correlated runtime readiness | 28.77 / 562.62 | 31.78 / 538.64 | +| Native marker verification | 75.07 / 168.00 | 72.32 / 546.11 | +| Complete navigation | 753.13 / 812.97 | 274.20 / 788.37 | +| Worker startup | 203.00 / 215.85 | 201.50 / 212.85 | +| Worker body | 505.18 / 533.74 | 510.90 / 533.80 | +| Settled worker | 723.34 / 750.74 | 729.28 / 758.36 | +| Worker settlement | 18.12 / 20.96 | 19.63 / 21.90 | +| Receipts | 2.62 / 2.93 | 2.67 / 2.98 | +| Session cleanup | 115.24 / 126.77 | 118.25 / 126.46 | +| Diagnostic native reads | 13.33 / 18.08 | 7.81 / 14.71 | +| Extraction | 6.50 / 10.34 | 6.50 / 8.67 | + +| Fabric total | Baseline | Candidate | +| -------------------------- | ------------------- | ------------------- | +| Runner p50 / p95 ms | 7939.07 / 8845.37 | 6680.04 / 6995.31 | +| Runner aggregate ms | 63950.71 | 53173.56 | +| Runner sample variance ms² | 306828.83 | 91557.04 | +| Runner range ms | 7478.77–9245.48 | 6192.64–7031.09 | +| Lifecycle p50 / p95 ms | 19887.17 / 28433.88 | 18461.08 / 28733.43 | +| Lifecycle aggregate ms | 171994.83 | 162636.05 | +| Lifecycle variance ms² | 19950017.41 | 30801394.39 | + +The bounded Fabric runner median was 15.9% lower. Matching repetitions by +index gave a median candidate-minus-baseline delta of −1141.99 ms; a +10,000-resample paired-median bootstrap interval was +−1887.49 to −782.75 ms (95%, fixed seed). This does not account for +temporal/system-load bias or establish reliability. Navigation's median +improved while native-verification p95 worsened; no universal tail-latency +or startup claim follows. This single-source cohort parsed once per run on +both sides and does not measure the cross-story cache benefit. + +### Paper before/after + +The same eight-plus-eight design, callback selection/order, and accessible +lane passed 24 callbacks and eight declared unsupported on each side. +Values are p50 / p95 milliseconds, with 24 callback samples or eight +extraction/total samples. + +| Phase | Baseline | Candidate | +| ---------------------------- | ----------------- | ----------------- | +| Session attach | 1094.93 / 1609.26 | 1434.73 / 1621.30 | +| Manifest | 2.60 / 3.14 | 2.69 / 3.11 | +| Input policy | 0.01 / 0.04 | 0.01 / 0.01 | +| Preparation | 0.13 / 0.24 | 0.14 / 0.20 | +| Upstream HTTP | 503.08 / 503.95 | Not dispatched | +| Correlated runtime readiness | 25.86 / 458.94 | 26.28 / 456.76 | +| Native marker verification | 97.40 / 170.07 | 167.26 / 582.90 | +| Complete navigation | 756.77 / 792.62 | 340.59 / 787.83 | +| Worker startup | 196.50 / 207.40 | 193.00 / 201.85 | +| Worker body | 523.32 / 556.19 | 529.44 / 547.30 | +| Settled worker | 736.00 / 789.32 | 741.24 / 756.44 | +| Worker settlement | 15.94 / 18.23 | 16.09 / 17.49 | +| Receipts | 2.46 / 2.76 | 2.44 / 2.62 | +| Session cleanup | 108.93 / 115.69 | 106.57 / 114.07 | +| Diagnostic native reads | 6.79 / 9.11 | 7.09 / 12.84 | +| Extraction | 6.44 / 8.97 | 6.64 / 9.77 | + +| Paper total | Baseline | Candidate | +| -------------------------- | ------------------- | ------------------- | +| Runner p50 / p95 ms | 7915.99 / 8755.09 | 7218.78 / 7811.59 | +| Runner aggregate ms | 60909.13 | 57610.90 | +| Runner sample variance ms² | 2199477.39 | 210172.97 | +| Runner range ms | 4123.59–8794.34 | 6440.61–7996.85 | +| Lifecycle p50 / p95 ms | 19928.62 / 62342.92 | 18921.30 / 60994.39 | +| Lifecycle aggregate ms | 224024.71 | 215170.59 | +| Lifecycle variance ms² | 529506165.46 | 519947445.91 | + +The bounded Paper runner median was 8.8% lower. Paired-by-index median +candidate-minus-baseline delta was −760.17 ms; its 10,000-resample bootstrap +95% interval was −1120.33 to −192.13 ms. One pair regressed by 3245.12 ms; +no samples were dropped. Cold first-lifecycle totals, including startup, were +retained on both sides (baseline 84945.44 ms, candidate 83318.31 ms). +Session-attach and native +verification medians worsened despite faster median navigation. The same +non-randomization, small-cohort, system-load, reliability, and cache-benefit +limits apply as for Fabric. + +### Candidate lifecycle parity + +Across each renderer's 24 passing candidate callbacks, keyed-preview/context +generations remained 2/3/4 per lifecycle. Fabric matched its native marker on +the first poll in all 24 callbacks, as did both renderer baselines. Paper's +candidate needed one poll in 12 callbacks and two polls in 12, contributing +to its slower native verification; fresh verification was never bypassed. +All eight lifecycles per side had distinct +verified app incarnations, one authenticated socket per lifecycle, and no +within-lifecycle app replacements or extra accepted hellos during callbacks. +Fabric accepted one hello and Paper two on the same connection. No independent +JS-init/HMR counter was added, and keyed generations do not count React +effect mounts. + +Candidate activity retained zero activation commands, provider activations, +application activation/deactivation, main/focused-window changes, or window +creation in the bounded observational cohort. Generic AX element-destruction +counts matched baseline (Fabric 96; Paper 240), not closed-window counts. +Sixteen Fabric and 17 Paper candidate callback samples had truncated event +history, versus 16 each on baseline. These are advisory observations, not +proof that no activation or visual flash occurred. + +### Window-only recording limits + +Separate synchronized recording lifecycles used only the target Storybook +window, outside the performance cohorts. All recordings were 1864×1564: + +| Renderer / revision | Duration seconds | Encoded frames | Last encoded timestamp seconds | Uniform-content candidates | +| ------------------- | ---------------: | -------------: | -----------------------------: | -------------------------: | +| Fabric baseline | 8.037 | 4 | 0.102 | 0 | +| Fabric candidate | 7.273 | 4 | 0.103 | 0 | +| Paper baseline | 8.590 | 4 | 0.103 | 0 | +| Paper candidate | 7.862 | 4 | 0.100 | 0 | + +The candidate detector measured near-uniform luma in a padded window-content +crop; it is **not a validated blank-preview or activation-flash classifier**. +The crop includes persistent Storybook chrome, variable-rate recordings hold +their last frame, and window-only pixels cannot establish foreground/window +raising behavior. There is **no measured flash-reduction claim**. Raw media +and derived image samples remain local, ignored artifacts. + +Phase 2 therefore ships measurement and a source-traced compatible proposal, +not an activation change. Phase 3's investigation gate is eligible because +attach is material, but pooling/startup changes remain deferred without their +own benefit and ownership/settlement evidence. Phase 4, new isolation +descriptors, preview reuse, and runtime protocol expansion remain excluded. + +### Focused native qualification and cache ownership + +The Fabric full-index accessible run rendered all 223 real catalog stories +and passed five selected callbacks (228 canonical passes, no failures or +unsupported results; run `57cf8594-addf-4cd3-9e60-5e1907a46b09`). +Selected stories were Button default, Dropdown default, and modern Callout +default/local-point/local-rectangle. This is not a full authored-test-suite +pass. The selected callbacks produced three source extractions and two +same-source cache hits across smoke's separate per-story runner invocations. +Each callback still had its own session, worker, and preview/context +generation (224–228); the application incarnation and authenticated +connection stayed stable. + +Fabric physical qualification also rendered all 223 stories and passed +15 selected callbacks (238 canonical passes, no failures or unsupported +results; run `6ba319db-cada-4e66-a000-513015b2610b`). Selection covered +Button default, Dropdown default, modern Callout default, and native +Callout default/window-commands. Four source extractions and one cache hit +were recorded. Dropdown's open-list keyboard case preceded its +initially-collapsed semantic case; fresh per-case remounting prevented the +open popup from contaminating the latter. Native and modern Callout +dismiss/reopen, outside-click, anchor input, and Escape cases passed. +This is sparse physical regression evidence, not the full physical suite +or a qualification of concurrent physical-input fixes on another branch. + +Paper passed the same full-index selections: 223 rendered stories plus five +accessible callbacks (228 canonical passes; run +`c6e2032d-76ee-4021-bc96-3e1c73be60ff`) and 223 plus 15 physical callbacks +(238 passes; run `9351f32d-29fe-4e6e-ae3d-cadd85a51989`). Neither lane had +failures, skips, unsupported callbacks, retries, or incomplete work. +Source extraction/cache-hit counts matched Fabric (3/2 accessible, 4/1 +physical). Fresh callback generations were 224–228 and 224–238 respectively; +each lane retained one verified app incarnation and authenticated socket, +with two accepted hellos. + +Fabric's three accessible Callout callbacks each observed two activation +commands +and two provider activations during their worker interval, plus focused-window +changes. This shows that window-oriented test helpers can activate even +without physical input; it does not identify a painted flash or justify +removing foreground guarantees. The timing cohort itself observed no +activation commands on either side. + +### Package validation + +| Workspace / command | Result | +| ---------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| Storybook integration: `yarn format && yarn lint && yarn build && yarn test --runInBand` | **Passed**; 405 tests passed, four skipped, 28 suites | +| Driver: `yarn format && yarn lint && yarn build` | **Passed** | +| Driver portable: `FURN_NATIVE_DRIVER_TEST=0 yarn test --runInBand` | **Passed**; 1223 tests passed, 66 skipped, 73 suites passed and eight skipped | +| Root: `yarn oxfmt --check` | **Passed** | +| Root: `yarn check-agent-instructions` | **Passed** | +| Root: `yarn change:check` after committing patch changesets | **Passed** | + +New regressions cover direct dispatch without HTTP, normal args versus +protected args refusal, correlated/stale/foreign readiness, uncertainty after +dispatch cancellation or disconnect, native-marker verification, bounded +diagnostic projection, source-content/location invalidation, run ownership, +cross-invocation cache hits, and manifest-digest rejection even on a cache hit. + +### Final native and aggregate validation + +All native/GUI and aggregate commands were serialized through the shared +native lock, with project-local temporary files. + +From `apps/storybook`, the strict standalone-helper build **passed**: + +```sh +swift build --package-path ../../packages/agentic/desktop-driver/native/macos --scratch-path artifacts/refresh-reduction/native-strict-build -c release -Xswiftc -warnings-as-errors +artifacts/refresh-reduction/native-strict-build/release/furn-desktop-driver-host --self-test +``` + +The self-test **passed 448/448 checks**. From the driver workspace, +`FURN_NATIVE_DRIVER_TEST=1 yarn test nativeDriver nativeCache applicationIncarnation serviceIdentity --runInBand` +**passed 87 tests**, with 16 skipped, six suites passed and two Windows suites +skipped. No product Swift or React-module boundary changed; Office +module/bridging-header qualification is not applicable. + +The single cache-cleared `yarn clean-lage-cache && yarn lage buildci` attempt +**failed** in scripts' negative instruction fixtures: a temporary directory +inside ignored artifacts inherited the outer Git repository. The scripts +workspace's `yarn test` then **passed all 76 tests** with Git discovery +bounded by the temporary directory's parent. The next plain aggregate +**failed** two existing timing-sensitive driver tests in parallel execution +(`boundedCommand` termination signal and `privateStorage` timeout). +`FURN_NATIVE_DRIVER_TEST=0 yarn test boundedCommand privateStorage --runInBand` +then **passed 73 tests**, with ten skipped. + +The final plain aggregate **passed** using the repository's existing +`TF_BUILD` serial-Jest mode and the fixture environment below, without another +cache clear: + +```sh +export TMPDIR="$PWD/apps/storybook/artifacts/refresh-reduction/tmp" +export GIT_CEILING_DIRECTORIES="$PWD/apps/storybook/artifacts/refresh-reduction" +TF_BUILD=1 yarn lage buildci +``` + +Lage reported 46 successful and 132 skipped tasks, zero pending/aborted, exit +zero. These task skips are not test-pass counts. No source changes were made +to the instruction validator, private storage, or bounded process runner. From 389a518ee9ffbdc3164295864855222427685b82 Mon Sep 17 00:00:00 2001 From: Jason Morse Date: Thu, 12 Nov 2026 18:49:27 +0000 Subject: [PATCH 5/5] Fix desktop extraction cache and diagnostic review findings Revalidate observed helper dependencies on cache hits, separate process history from per-case event loss, remove synthetic mount counts, and document failed-attempt timings. Record plain parallel aggregate failures reproduced on the stacked base. Part of #4435 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .changeset/calm-story-observations.md | 2 +- .../references/current-status.md | 51 ++++++-- .../config/transform-story-tests.cjs | 9 +- .../config/transform-story-tests.d.cts | 1 + .../references/contributing.md | 4 + .../references/current-status.md | 121 +++++++++++++++--- .../references/story-tests.md | 31 +++-- .../src/testing/extractWdioTests.ts | 14 +- .../src/testing/runDiagnostics.test.ts | 47 ++++++- .../src/testing/runDiagnostics.ts | 10 +- .../src/testing/runWdioTests.ts | 1 - .../src/testing/wdioExtractionCache.test.ts | 32 +++++ .../src/testing/wdioExtractionCache.ts | 30 ++++- .../src/testing/wdioRunner.contract.cjs | 6 +- 14 files changed, 303 insertions(+), 56 deletions(-) diff --git a/.changeset/calm-story-observations.md b/.changeset/calm-story-observations.md index 4ae834711a..6d57469838 100644 --- a/.changeset/calm-story-observations.md +++ b/.changeset/calm-story-observations.md @@ -5,4 +5,4 @@ Add bounded, service-local story selection timings and per-case execution/lifecycle observations without changing authenticated readiness, preview remounts, or worker/session isolation. Part of #4435. -Use authenticated direct navigation for driver-owned story selection and cache complete extracted test maps by source content within the owning run. Preserve normal args support, protected args refusal, manifest-digest verification, and the legacy HTTP default for direct library consumers. +Use authenticated direct navigation for driver-owned story selection and cache complete extracted test maps by source content and observed helper digests within the owning run. Preserve normal args support, protected args refusal, manifest-digest verification, and the legacy HTTP default for direct library consumers. Report process-history truncation separately from possible per-case loss; preview generations infer remounts rather than independently counting them. diff --git a/packages/agentic/desktop-driver/references/current-status.md b/packages/agentic/desktop-driver/references/current-status.md index b2d0a1d5a8..0ee92a0ac3 100644 --- a/packages/agentic/desktop-driver/references/current-status.md +++ b/packages/agentic/desktop-driver/references/current-status.md @@ -485,13 +485,16 @@ are not copied into these observations. The instrumented baseline ran eight independently launched accessible cohorts per macOS renderer, each with three passing ListboxItem callbacks and one declared macOS-unsupported callback. Each passing callback selected a fresh -keyed preview/context generation; every cohort retained one verified app +preview/context generation; keyed remounts are inferred from rendering +policy, not independently counted. Every cohort retained one verified app incarnation and one authenticated bridge connection during its callbacks. JS initialization/HMR was not observed independently and cannot be inferred from hello counts. Fabric accepted one hello and Paper two on the same socket. No activation command or main/focused-window change was observed in these -baseline cohorts; Paper recorded one application deactivation, and 16 of 24 -event samples per renderer carried truncation warnings. A zero advisory count +baseline cohorts; Paper recorded one application deactivation. Historical +per-case truncation counts are withdrawn: a process-lifetime history flag +and globally allocated sequence gaps did not establish per-case event loss. +A zero advisory count is not visual qualification. The Storybook evidence ledger, `packages/agentic/storybook-desktop/references/current-status.md`, records @@ -534,11 +537,11 @@ qualification of another branch's physical-input corrections. The unchanged standalone macOS helper passed a release build with `-Xswiftc -warnings-as-errors` and all 448 self-test checks. Focused `FURN_NATIVE_DRIVER_TEST=1` contracts passed 87 tests, with 16 skipped -(Windows contracts not run locally). The final aggregate passed in the -repository's serial-Jest CI mode after isolating Git discovery for -project-local temporary fixtures and rerunning two timing-sensitive -parallel-test failures. The Storybook evidence ledger records exact commands, -failed attempts, package counts, and the final passed result. +(Windows contracts not run locally). An earlier local aggregate passed with `TF_BUILD=1` serializing package Jest +runs and with project-local fixture overrides; this is not PR CI mode. +The PR workflow uses parallel `yarn lage buildci` without those overrides. +The Storybook evidence ledger records exact commands, failed attempts, +package counts, and the subsequent plain-command review qualification. ### Activation trace and deferred flashing change @@ -565,3 +568,35 @@ occlusion refused. An unqualified dedupe must not fork those semantics. Zero activation counters in this read-only cohort do not meet the optimization gate. The compatible experiment and its qualification conditions are tracked in [remaining work](remaining-work.md#macos-activation-and-popup-polling). + +## 2026-11-12: refresh-diagnostic review corrections + +Part of #4435. The Storybook report now separates the native process-lifetime +`eventsTruncated` flag into `processEventsTruncated` from a possible +per-case `eventsTruncated` warning. A case warning requires truncated +history whose earliest retained event is newer than the case cursor. +Globally allocated sequence gaps are not loss evidence; the unsupported +command-truncation inference was removed. Historical 16/17-of-24 +comparisons are withdrawn because the original bounded reports cannot +reconstruct per-case loss. The duplicate mount-generation field was also +removed: fresh preview generations infer remounting but do not count mounts +independently. No native producer or activation behavior changed. + +The earlier `TF_BUILD=1` aggregate was local serialization, not PR CI mode. +Under the shared native lock, plain +`yarn clean-lage-cache && yarn lage buildci` with no environment overrides +**failed in both candidate runs** on the `boundedCommand` termination-signal +assertion; `privateStorage` passed in both. The identical +command on independently installed, detached base `aadc2f359` **failed** +that same assertion and a `privateStorage` timeout. The base worktree was +removed after terminating its verified orphaned Jest worker. Those +pre-existing tests and implementations were not changed. + +Review regressions **passed 112 tests** across four Storybook suites. +macOS Paper manifest/bundle and full-index accessible smoke **passed** +(223 rendered stories plus five callbacks, three extractions/two cache +hits). A harness PATH failure before selection was corrected by invoking +it through Yarn, without a product change. macOS Fabric and physical +lanes were **not rerun for this follow-up**; Windows Fabric and Win32 Paper +were **not run locally**. The earlier paired timings and window recordings +were not repeated, and no new performance or flashing claim is made. diff --git a/packages/agentic/storybook-desktop/config/transform-story-tests.cjs b/packages/agentic/storybook-desktop/config/transform-story-tests.cjs index 6952c743b1..d86025b886 100644 --- a/packages/agentic/storybook-desktop/config/transform-story-tests.cjs +++ b/packages/agentic/storybook-desktop/config/transform-story-tests.cjs @@ -404,8 +404,10 @@ function importedHelper(callee, fromFile) { function readHelperModule(file, context) { if (context.modules.has(file)) return context.modules.get(file); let helper = null; + let source; try { - const ast = parseSync(fs.readFileSync(file, 'utf8'), { + source = fs.readFileSync(file, 'utf8'); + const ast = parseSync(source, { babelrc: false, configFile: false, filename: file, @@ -446,6 +448,7 @@ function readHelperModule(file, context) { // A helper this check cannot parse is left to the run-time policy, like any other input it cannot see. helper = null; } + context.onReadHelper?.(file, source); context.modules.set(file, helper); return helper; } @@ -503,9 +506,9 @@ function findPhysicalInput(root, file, context, helper) { * refused by the accessible lane's input policy when the case runs, and the case's receipts are checked against its * declaration in every lane. */ -function rejectDirectInput(callback, { name, input, inputSource, sourceFile }) { +function rejectDirectInput(callback, { name, input, inputSource, sourceFile, onReadHelper }) { if (input === 'physical') return; - const finding = findPhysicalInput(callback, sourceFile, { modules: new Map(), functions: new Map() }); + const finding = findPhysicalInput(callback, sourceFile, { modules: new Map(), functions: new Map(), onReadHelper }); if (!finding) return; const subject = name ? `"${name}"` : 'This case'; const declared = inputSource === 'declared'; diff --git a/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts b/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts index d9d464e526..f298a83992 100644 --- a/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts +++ b/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts @@ -24,6 +24,7 @@ declare namespace storyTests { inputSource: 'declared' | 'derived'; /** The story file, so helpers imported from relative modules are followed. */ sourceFile?: string; + onReadHelper?: (file: string, source: string | undefined) => void; }, ): void; } diff --git a/packages/agentic/storybook-desktop/references/contributing.md b/packages/agentic/storybook-desktop/references/contributing.md index 0543677c64..3f582426c6 100644 --- a/packages/agentic/storybook-desktop/references/contributing.md +++ b/packages/agentic/storybook-desktop/references/contributing.md @@ -94,6 +94,10 @@ copy them here. `src/cli/inputLane.ts` resolves the requested lane, `config/transform-story-tests.cjs` is both the Metro Babel plugin and the shared AST library: its `rejectDirectInput` check runs from `src/testing/extractWdioTests.ts` only, because the Metro transform has no validated descriptor to judge by. +Run-owned extraction entries retain digests for every helper module that +check reads, including transitive helpers and unreadable files. Recheck those +dependencies on a cache hit; a source-file digest alone cannot preserve the +direct-input policy when helpers change. Keep story callbacks statically extractable and authentication checks fail-closed. Update references when commands, supported config, outputs, or diff --git a/packages/agentic/storybook-desktop/references/current-status.md b/packages/agentic/storybook-desktop/references/current-status.md index 595880e12d..31f24a1192 100644 --- a/packages/agentic/storybook-desktop/references/current-status.md +++ b/packages/agentic/storybook-desktop/references/current-status.md @@ -69,9 +69,11 @@ attributed to those differences. Each lifecycle had one verified PID/start-time pair and one authenticated bridge connection. Fabric accepted one hello per baseline lifecycle; Paper accepted -two on the same connection. Each callback selected a fresh generation -(2, 3, 4 after the traversal's first generation), with matching keyed-mount and -context generations. No within-lifecycle application replacement or new bridge +two on the same connection. Each callback selected a fresh preview/context +generation (2, 3, 4 after the traversal's first generation). The historical +`mountGeneration` field merely copied that value; keyed remounting was +inferred from the rendering policy, not independently observed. That derived +field has been removed. No within-lifecycle application replacement or new bridge connection was observed. JS initialization/HMR is explicitly **not observed**: neither a hello nor the absence of a new socket proves a JS initialization count. The keyed generation is not an instrumented React effect-mount counter. @@ -79,8 +81,9 @@ The keyed generation is not an instrumented React effect-mount counter. Native activity reports retained fixed counters only. No activation command, provider activation, application activation, or main/focused-window change was observed in these baseline cohorts; Paper recorded one application -deactivation. Sixteen of 24 callback event samples per renderer carried a -truncation warning. `AXUIElementDestroyed` +deactivation. The original truncation projection combined a process-lifetime +flag with invalid sequence-gap inference; its per-case truncation counts are +withdrawn and cannot be reconstructed from these bounded reports. `AXUIElementDestroyed` notifications can describe preview elements, not necessarily closed windows. The bounded native event ring can be truncated, and its observations are advisory. These counters do not establish whether a user saw a flash. @@ -106,7 +109,9 @@ Part of #4435. Candidate implementation: `06446c276`; instrumentation: selection without an upstream HTTP mutation. Controlled policy remains independent and still refuses protected args and foreign mutation relaying. Full extracted export maps are cached by source content/location within one -owning run, including across smoke's per-story calls; manifest digest checks, +owning run, including across smoke's per-story calls. Review corrections also +bind cache hits to digests of the helper files read by input-policy validation. +Manifest digest checks, remounts, fresh sessions, fresh workers, and native marker verification remain. ### Fabric before/after @@ -203,7 +208,7 @@ limits apply as for Fabric. ### Candidate lifecycle parity -Across each renderer's 24 passing candidate callbacks, keyed-preview/context +Across each renderer's 24 passing candidate callbacks, preview/context generations remained 2/3/4 per lifecycle. Fabric matched its native marker on the first poll in all 24 callbacks, as did both renderer baselines. Paper's candidate needed one poll in 12 callbacks and two polls in 12, contributing @@ -219,8 +224,9 @@ Candidate activity retained zero activation commands, provider activations, application activation/deactivation, main/focused-window changes, or window creation in the bounded observational cohort. Generic AX element-destruction counts matched baseline (Fabric 96; Paper 240), not closed-window counts. -Sixteen Fabric and 17 Paper candidate callback samples had truncated event -history, versus 16 each on baseline. These are advisory observations, not +The earlier 16/17-of-24 truncation comparison was invalid: sticky +process-history flags and globally allocated sequence gaps do not establish +per-case loss. That comparison is withdrawn. These are advisory observations, not proof that no activation or visual flash occurred. ### Window-only recording limits @@ -257,7 +263,7 @@ Selected stories were Button default, Dropdown default, and modern Callout default/local-point/local-rectangle. This is not a full authored-test-suite pass. The selected callbacks produced three source extractions and two same-source cache hits across smoke's separate per-story runner invocations. -Each callback still had its own session, worker, and preview/context +Each callback still had its own session, worker, and fresh preview/context generation (224–228); the application incarnation and authenticated connection stayed stable. @@ -308,7 +314,7 @@ dispatch cancellation or disconnect, native-marker verification, bounded diagnostic projection, source-content/location invalidation, run ownership, cross-invocation cache hits, and manifest-digest rejection even on a cache hit. -### Final native and aggregate validation +### Initial native and environment-overridden aggregate validation All native/GUI and aggregate commands were serialized through the shared native lock, with project-local temporary files. @@ -326,19 +332,20 @@ The self-test **passed 448/448 checks**. From the driver workspace, skipped. No product Swift or React-module boundary changed; Office module/bridging-header qualification is not applicable. -The single cache-cleared `yarn clean-lage-cache && yarn lage buildci` attempt +The initial cache-cleared `yarn clean-lage-cache && yarn lage buildci` attempt **failed** in scripts' negative instruction fixtures: a temporary directory inside ignored artifacts inherited the outer Git repository. The scripts workspace's `yarn test` then **passed all 76 tests** with Git discovery -bounded by the temporary directory's parent. The next plain aggregate -**failed** two existing timing-sensitive driver tests in parallel execution +bounded by the temporary directory's parent. The next environment-overridden aggregate +**failed** two timing-sensitive driver tests in parallel execution (`boundedCommand` termination signal and `privateStorage` timeout). `FURN_NATIVE_DRIVER_TEST=0 yarn test boundedCommand privateStorage --runInBand` then **passed 73 tests**, with ten skipped. -The final plain aggregate **passed** using the repository's existing -`TF_BUILD` serial-Jest mode and the fixture environment below, without another -cache clear: +An earlier local aggregate **passed** with `TF_BUILD=1` serializing each +package's Jest run and with the fixture environment below, without another +cache clear. This was **not PR CI mode**: `.github/workflows/pr.yml` runs +parallel `yarn lage buildci` without these overrides. ```sh export TMPDIR="$PWD/apps/storybook/artifacts/refresh-reduction/tmp" @@ -349,3 +356,83 @@ TF_BUILD=1 yarn lage buildci Lage reported 46 successful and 132 skipped tasks, zero pending/aborted, exit zero. These task skips are not test-pass counts. No source changes were made to the instruction validator, private storage, or bounded process runner. + +## 2026-11-12: refresh-reduction review corrections + +Part of #4435. This follow-up corrects the run-owned extraction cache and +diagnostic interpretation without changing dispatch, per-case workers, +context verification, activation, or native protocol semantics. + +- Cache entries now retain content digests for every helper module read by + direct-input validation, including transitive helpers and failed reads. + Hits recheck those digests; a changed helper reruns extraction and policy + validation even when the story source is unchanged. Regressions change a + direct or transitive helper to keyboard input after a confirmed cache hit. +- `processEventsTruncated` reports the native process-lifetime flag. + `eventsTruncated` reports possible per-case history loss only when a + truncated history no longer covers the case cursor. Globally allocated + sequence gaps alone do not establish loss. The unsupported + `commandsTruncated` inference was removed. Tests cover sticky lifetime + flags, global gaps, and lost cursor coverage. Historical 16/17-of-24 + truncation comparisons above are withdrawn, not relabeled: the original + bounded projections cannot reconstruct actual per-case loss. +- The duplicate `mountGeneration` field was removed. Context/preview + generations are observed; keyed remounts are inferred from the default + rendering policy, not independently counted. A runner contract requires + the derived field to be absent. +- Present phase durations include failed attempts because measurement runs + in `finally`. Missing durations do not establish success or zero cost. + A runner regression verifies attempted navigation time on failure and + absence of worker time when the worker was never started. + +### Review qualification + +The regression-first run failed as expected before these fixes. From the +Storybook integration workspace, +`yarn test extractWdioTests wdioExtractionCache runDiagnostics wdioRunner --runInBand` +then **passed 112 tests in four suites**, including a repeat with the normal +inherited environment. Package format/lint/build passed. + +From `apps/storybook`, `yarn storybook manifest --macos --paper` and +`yarn storybook bundle --macos --paper` **passed**. A full-index CLI-API smoke +with the app-owned configuration, Paper renderer, accessible input, and the +Button/Dropdown/modern-Callout selection **passed 228 checks**: 223 rendered +stories and five callbacks; ten physical callbacks were excluded by input +policy. The five callback reports showed three source extractions, two +cross-invocation cache hits, no synthetic mount field, and separate process +and case history flags. The first local harness invocation failed before +selection because plain `node` did not provision Yarn's tool PATH +(`spawn rnx-cli ENOENT`); rerunning through `yarn node` passed without a +product change. + +The plain aggregate commands below ran under the shared native lock with +the normal inherited environment: no `TF_BUILD`, `TMPDIR`, or +`GIT_CEILING_DIRECTORIES` overrides. Unlike the earlier locally serialized +run, they preserve parallel Jest execution. Local Node was 24.18.0; the PR +workflow declares Node 22, so this is not complete hosted-CI parity. + +| Scope | Exact command | Result | +| -------------------------- | -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | +| Candidate, first plain run | `yarn clean-lage-cache && yarn lage buildci` | **Failed**: `boundedCommand` expected `SIGKILL`, received `SIGTERM`; 122 Lage tasks succeeded, one failed | +| Detached base `aadc2f359` | `yarn clean-lage-cache && yarn lage buildci` | **Failed**: the same `boundedCommand` assertion plus `privateStorage`'s 10-second test timeout; 129 Lage tasks succeeded, one failed | + +A final plain repeat of the same cache-cleared candidate command also +**failed** the `boundedCommand` assertion (164 Lage tasks succeeded, one +failed). `privateStorage` passed in both candidate runs. In each candidate +driver suite, 1222 tests passed, one failed, and 66 were skipped. +`yarn oxfmt --check`, `yarn check-agent-instructions`, and +`yarn markdown-link-check -c .github/markdown-link-check-config.json ` +for every changed Markdown file **passed**. + +The detached base was installed independently after its first command +reported missing dependencies; it did not reuse candidate workspace +modules or outputs. This reproduces the aggregate failures before the +refresh-reduction commits. Neither unrelated test nor implementation was +changed. The base worktree was removed, and its orphaned Jest worker was +terminated by its verified PID. + +The paired performance and recording evidence above remains evidence for +`06446c276`, not a fresh benchmark of this correction. No new speedup or +flash-reduction claim is made. macOS Fabric and physical-input lanes were +**not rerun in this follow-up**; their earlier qualifications remain +historical. Windows Fabric and Win32 Paper were **not run locally**. diff --git a/packages/agentic/storybook-desktop/references/story-tests.md b/packages/agentic/storybook-desktop/references/story-tests.md index de9babdff2..f61190df3b 100644 --- a/packages/agentic/storybook-desktop/references/story-tests.md +++ b/packages/agentic/storybook-desktop/references/story-tests.md @@ -174,7 +174,9 @@ Part of #4435. The linked **Executable story results** `run.json` artifact records `diagnostics` without changing test selection, readiness, or isolation. Run-level fields are `extractionMs`, `sourceExtractions`, `sourceCacheHits`, and `totalMs`. Extraction counts are full-file parses; cache hits still include -source rereading/hashing and manifest-digest verification. +source rereading/hashing, digest checks for helper modules read by the +direct-input policy, and manifest-digest verification. A changed helper, +including a transitive helper, invalidates the extraction and reruns the policy. Each attempted case records `diagnostics.phases`: - `sessionAttachMs`, `manifestMs`, and `policyMs`; @@ -190,21 +192,30 @@ Readiness is elapsed from the start of preparation; HTTP timing starts at its dispatch. Do not sum those fields. Worker startup includes spec preparation, Node/import startup, reporter setup, and attachment; body timing surrounds the callback. Settlement is the remaining supervisor-observed worker time, including -process exit and stop. Missing fields mean the phase did not complete or its -producer could not observe it, never a zero-cost pass. - -`diagnostics.lifecycle` records the context and mount generations (equal under -the default keyed remount), native marker poll count, authenticated hello and -connection counters, and the native-verified application PID/start time when -available. `jsInitializations: 'not-observed'` is explicit: a bridge hello can +process exit and stop. Durations recorded in `finally` include failed +attempts; a present field does not mean that phase succeeded. Missing fields +mean the phase was not reached or its producer did not report an observation, +never a zero-cost pass. + +`diagnostics.lifecycle` records the context generation, native marker poll +count, authenticated hello and connection counters, and the native-verified +application PID/start time when available. The context generation is the +preview generation; a keyed remount is inferred from that generation and the +default rendering policy, not independently observed. No separate mount +counter is reported. `jsInitializations: 'not-observed'` is explicit: a bridge hello can answer another challenge without initializing JavaScript. No runtime message, descriptor, or preview-reuse protocol is added. `windowActivity` reduces the existing native diagnostic rings to per-case notification and activation-command counters. It never retains native handles, window titles, argument values, or raw command details. Unsupported or failed -reads are explicit. `observing`, registration-failure counts, and truncation -flags limit interpretation: these are advisory lower bounds, not proof of +reads are explicit. `processEventsTruncated` is the native process-lifetime +history flag, not a per-case loss count. `eventsTruncated` warns of possible +case-history loss only when that history is truncated and its earliest +retained event is newer than the case cursor. Global sequence gaps alone +do not establish loss, and no command-truncation flag is inferred. +`observing`, registration failures, and these history limits make the counters +advisory lower bounds, not proof of foreground ownership or visible flashing. Target-window-only recording is a separate, permission-gated qualification; no recording is enabled by the runner. The observation reads themselves are included in `activityReadMs`. diff --git a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts index eb18f658fd..1e478ae619 100644 --- a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts +++ b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts @@ -13,7 +13,11 @@ export type ExtractedWdioStory = { tests: readonly { name?: string; code: string; options?: ResolvedDesktopCaseOptions }[]; }; -export function extractWdioTests(code: string, sourceFile: string): ReadonlyMap { +export function extractWdioTests( + code: string, + sourceFile: string, + onReadHelper?: (file: string, source: string | undefined) => void, +): ReadonlyMap { const tests = new Map(); transformSync(code, { babelrc: false, @@ -30,7 +34,13 @@ export function extractWdioTests(code: string, sourceFile: string): ReadonlyMap< const compiledCode = compileWdioTest(callback, sourceFile, code); const descriptor = options === undefined ? undefined : validateDesktopCaseOptions(options, `${sourceFile}.wdio`); if (descriptor !== undefined) { - rejectDirectInput(callback, { name, input: descriptor.input, inputSource: descriptor.inputSource, sourceFile }); + rejectDirectInput(callback, { + name, + input: descriptor.input, + inputSource: descriptor.inputSource, + sourceFile, + onReadHelper, + }); } return { ...(name === undefined ? {} : { name }), diff --git a/packages/agentic/storybook-desktop/src/testing/runDiagnostics.test.ts b/packages/agentic/storybook-desktop/src/testing/runDiagnostics.test.ts index 399dfcde67..362096a477 100644 --- a/packages/agentic/storybook-desktop/src/testing/runDiagnostics.test.ts +++ b/packages/agentic/storybook-desktop/src/testing/runDiagnostics.test.ts @@ -46,13 +46,13 @@ test('reduces native activity to bounded counters, never window titles or comman elementsDestroyed: 0, activationCommands: 1, providerActivations: 2, - commandsTruncated: true, eventsTruncated: true, + processEventsTruncated: true, }); expect(JSON.stringify(activity)).not.toContain('private'); }); -test('activity counters use per-case sequence deltas, preserving advisory truncation', () => { +test('activity counters use per-case sequence deltas', () => { expect( summarizeWindowActivity( { @@ -75,11 +75,52 @@ test('activity counters use per-case sequence deltas, preserving advisory trunca mainWindowChanges: 0, elementsDestroyed: 1, activationCommands: 0, - commandsTruncated: false, eventsTruncated: false, + processEventsTruncated: false, }); }); +test.each([false, true])('sequence gaps do not imply case truncation with process history flag %s', (eventsTruncated) => { + const activity = summarizeWindowActivity( + { + source: 'AX', + observing: true, + eventsTruncated, + events: [ + { ageMs: 2, element: 'window', notification: 'AXMainWindowChanged', sequence: 10 }, + { ageMs: 0, element: 'window', notification: 'AXWindowCreated', sequence: 30 }, + ], + commands: [ + { ageMs: 2, command: 'activate', outcome: 'ok', sequence: 20 }, + { ageMs: 0, command: 'activate', outcome: 'ok', sequence: 40 }, + ], + }, + { command: 20, event: 10 }, + ); + expect(activity).toMatchObject({ + events: 1, + windowsCreated: 1, + activationCommands: 1, + eventsTruncated: false, + processEventsTruncated: eventsTruncated, + }); + expect(activity).not.toHaveProperty('commandsTruncated'); +}); + +test('case event truncation requires lost cursor coverage in a truncated history', () => { + expect( + summarizeWindowActivity( + { + source: 'AX', + observing: true, + eventsTruncated: true, + events: [{ ageMs: 0, element: 'window', notification: 'AXWindowCreated', sequence: 30 }], + }, + { command: 20, event: 10 }, + ), + ).toMatchObject({ eventsTruncated: true, processEventsTruncated: true }); +}); + test('worker timing projection rejects nonnumeric and unbounded values without retaining extra fields', () => { expect(readWorkerTimings({ workerStartupMs: 10, workerBodyMs: 20, nonce: 'private' })).toEqual({ workerStartupMs: 10, diff --git a/packages/agentic/storybook-desktop/src/testing/runDiagnostics.ts b/packages/agentic/storybook-desktop/src/testing/runDiagnostics.ts index e1a8b44aab..c7bd74d69d 100644 --- a/packages/agentic/storybook-desktop/src/testing/runDiagnostics.ts +++ b/packages/agentic/storybook-desktop/src/testing/runDiagnostics.ts @@ -24,7 +24,6 @@ export type WdioCaseDiagnostics = { >; lifecycle?: { contextGeneration: number; - mountGeneration: number; nativeMarkerPolls?: number; authenticatedHellos?: number; bridgeConnections?: number; @@ -79,8 +78,8 @@ export type WindowActivity = elementsDestroyed: number; activationCommands: number; providerActivations: number; - commandsTruncated: boolean; eventsTruncated: boolean; + processEventsTruncated: boolean; }; export function windowActivityCursor(log: WindowLog): WindowActivityCursor { @@ -90,7 +89,8 @@ export function windowActivityCursor(log: WindowLog): WindowActivityCursor { /** Project only fixed counters; the native log also contains application text and must not enter the run report. */ export function summarizeWindowActivity(log: WindowLog, since: WindowActivityCursor = { command: 0, event: 0 }): WindowActivity { const commands = (log.commands ?? []).slice(-1024).filter(({ sequence }) => sequence > since.command); - const events = log.events.slice(-1024).filter(({ sequence }) => sequence > since.event); + const retainedEvents = log.events.slice(-1024); + const events = retainedEvents.filter(({ sequence }) => sequence > since.event); return { status: 'observed', events: events.length, @@ -106,7 +106,7 @@ export function summarizeWindowActivity(log: WindowLog, since: WindowActivityCur providerActivations: commands.filter(({ detail }) => ['already-frontmost', 'accessibility', 'workspace', 'accessibility+workspace'].includes(String(detail?.method)), ).length, - commandsTruncated: commands.length > 0 && commands[0].sequence > since.command + 1, - eventsTruncated: log.eventsTruncated || (events.length > 0 && events[0].sequence > since.event + 1), + eventsTruncated: (log.eventsTruncated || retainedEvents.length < log.events.length) && (retainedEvents[0]?.sequence ?? 0) > since.event, + processEventsTruncated: log.eventsTruncated, }; } diff --git a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts index 8746a7aed1..fe81b1f448 100644 --- a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts +++ b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts @@ -297,7 +297,6 @@ export async function runWdioStoryTests( const application = readApplicationIncarnation(observations?.application); diagnostics.lifecycle = { contextGeneration: ready.previewGeneration, - mountGeneration: ready.previewGeneration, jsInitializations: 'not-observed', ...(application ? { application } : {}), ...(observations?.nativeMarkerPolls === undefined ? {} : { nativeMarkerPolls: observations.nativeMarkerPolls }), diff --git a/packages/agentic/storybook-desktop/src/testing/wdioExtractionCache.test.ts b/packages/agentic/storybook-desktop/src/testing/wdioExtractionCache.test.ts index 6a2be0b079..42c122b84d 100644 --- a/packages/agentic/storybook-desktop/src/testing/wdioExtractionCache.test.ts +++ b/packages/agentic/storybook-desktop/src/testing/wdioExtractionCache.test.ts @@ -59,3 +59,35 @@ test('identical content in different source locations retains location-specific expect(other.cacheHit).toBe(false); expect(other.tests.get('First')!.tests[0].code).not.toBe(first.tests.get('First')!.tests[0].code); }); + +test.each(['direct', 'transitive'])('rechecks changed %s helper input without a story-file edit', (kind) => { + const helperFile = path.join(root, 'helper.ts'); + const leafFile = path.join(root, 'leaf.ts'); + fs.writeFileSync( + sourceFile, + `import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; + export const First = { wdio: [ + desktopIt('observes', { input: 'accessible' }, async ({ browser }) => { + const { probe } = await import('./helper.ts'); + await probe(browser); + }), + ] };`, + ); + const safeHelper = 'export async function probe(browser) { return browser; }'; + fs.writeFileSync(leafFile, safeHelper); + fs.writeFileSync( + helperFile, + kind === 'direct' + ? safeHelper + : `export async function probe(browser) { + const { probe } = await import('./leaf.ts'); + await probe(browser); + }`, + ); + const cache = new WdioExtractionCache(); + expect(cache.read(sourceFile).cacheHit).toBe(false); + expect(cache.read(sourceFile).cacheHit).toBe(true); + + fs.writeFileSync(kind === 'direct' ? helperFile : leafFile, `export async function probe(browser) { await browser.keys('Tab'); }`); + expect(() => cache.read(sourceFile)).toThrow('sends physical keyboard input'); +}); diff --git a/packages/agentic/storybook-desktop/src/testing/wdioExtractionCache.ts b/packages/agentic/storybook-desktop/src/testing/wdioExtractionCache.ts index 743d3a5616..1f2d4679a0 100644 --- a/packages/agentic/storybook-desktop/src/testing/wdioExtractionCache.ts +++ b/packages/agentic/storybook-desktop/src/testing/wdioExtractionCache.ts @@ -3,17 +3,37 @@ import fs from 'node:fs'; import { extractWdioTests, type ExtractedWdioStory } from './extractWdioTests.js'; +function sourceDigest(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +function helperDigest(file: string): string | undefined { + try { + return sourceDigest(fs.readFileSync(file, 'utf8')); + } catch { + return undefined; + } +} + /** Owned by one test/smoke run, with only the latest source identity retained for each file. */ export class WdioExtractionCache { - private readonly sources = new Map }>(); + private readonly sources = new Map< + string, + { digest: string; helpers: ReadonlyMap; tests: ReadonlyMap } + >(); read(sourceFile: string): { tests: ReadonlyMap; cacheHit: boolean } { const source = fs.readFileSync(sourceFile, 'utf8'); - const digest = createHash('sha256').update(source).digest('hex'); + const digest = sourceDigest(source); const cached = this.sources.get(sourceFile); - if (cached?.digest === digest) return { tests: cached.tests, cacheHit: true }; - const tests = extractWdioTests(source, sourceFile); - this.sources.set(sourceFile, { digest, tests }); + if (cached?.digest === digest && [...cached.helpers].every(([file, expected]) => helperDigest(file) === expected)) { + return { tests: cached.tests, cacheHit: true }; + } + const helpers = new Map(); + const tests = extractWdioTests(source, sourceFile, (file, content) => { + helpers.set(file, content === undefined ? undefined : sourceDigest(content)); + }); + this.sources.set(sourceFile, { digest, helpers, tests }); return { tests, cacheHit: false }; } } diff --git a/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs b/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs index 7388449cc4..9217410613 100644 --- a/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs +++ b/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs @@ -239,7 +239,7 @@ async function main() { } const passed = await run('pass'); assert.ok(passed[0].diagnostics.lifecycle.contextGeneration >= 1); - assert.equal(passed[0].diagnostics.lifecycle.mountGeneration, passed[0].diagnostics.lifecycle.contextGeneration); + assert.equal(Object.hasOwn(passed[0].diagnostics.lifecycle, 'mountGeneration'), false); assert.equal(passed[0].diagnostics.lifecycle.jsInitializations, 'not-observed'); for (const phase of [ 'sessionAttachMs', @@ -292,6 +292,10 @@ async function main() { if (!rejected) throw new Error(`Expected ${story} to fail.`); const report = JSON.parse(fs.readFileSync(path.join(root, 'artifacts/windows/wdio/run.json'), 'utf8')); if (report.tests[0].status !== 'failed') throw new Error(`${story} was not persisted as failed.`); + if (story === 'navigation-failure') { + assert.ok(report.tests[0].diagnostics.phases.navigationMs >= 0, 'Failed navigation retains its attempted duration.'); + assert.equal(Object.hasOwn(report.tests[0].diagnostics.phases, 'workerMs'), false, 'An unstarted phase has no duration.'); + } if (story === 'fail' && !report.tests[0].error.includes('intentional inline assertion')) { throw new Error('Compact reporter lost the worker assertion diagnostic.'); }