diff --git a/.github/workflows/windows-chat-focus.yml b/.github/workflows/windows-chat-focus.yml new file mode 100644 index 0000000..988583b --- /dev/null +++ b/.github/workflows/windows-chat-focus.yml @@ -0,0 +1,220 @@ +name: Windows chat focus E2E + +on: + workflow_dispatch: + inputs: + live_solheim: + description: Run real Solheim terminal/diff tests (requires environment approval) + type: boolean + default: false + live_runner: + description: Hosted Windows Server or an interactive self-hosted Windows 11 x64 runner + type: choice + options: + - windows-2022 + - windows11-x64 + default: windows-2022 + pull_request: + types: [opened, synchronize, reopened, labeled] + paths: + - ".github/workflows/windows-chat-focus.yml" + - "jetbrains_plugin/scripts/ui-harness/**" + - "jetbrains_plugin/src/main/kotlin/**" + - "jetbrains_plugin/build.gradle.kts" + - "jetbrains_plugin/gradle.properties" + - "extension_host/**" + +permissions: + contents: read + +concurrency: + # Skipped label events must not replace a pending build for this PR. + group: windows-chat-focus-${{ github.ref }}-${{ github.event.action == 'labeled' && github.event.label.name != 'live-solheim' && 'ignored-label' || 'validation' }} + cancel-in-progress: ${{ github.event.action != 'labeled' }} + +jobs: + build: + if: github.event.action != 'labeled' || github.event.label.name == 'live-solheim' + name: Build this checkout with the pinned extension + # Keep native dependency builds independent of ubuntu-latest image changes. + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + # Match the extension used in the existing focus investigation. + ZOO_EXTENSION_VERSION: "3.84.0" + # This job packages the plugin; headful tests use the IDE's bundled JCEF. + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" + steps: + - uses: actions/checkout@v5 + with: + submodules: recursive + persist-credentials: false + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "17" + cache: gradle + - uses: actions/setup-node@v6 + with: + node-version: "22" + cache: npm + cache-dependency-path: extension_host/package-lock.json + - name: Install native build dependencies + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends build-essential libx11-dev libxkbfile-dev pkg-config + - name: Validate the local streaming fixture and authenticated relay + working-directory: jetbrains_plugin/scripts/ui-harness + run: python3 -m unittest -v test_streaming_fixture.py test_solheim_proxy.py + - name: Prepare extension host source + run: ./scripts/setup.sh --skip-deps --verbose + - name: Cache the cross-platform VS Code runtime + uses: actions/cache@v4 + with: + path: jetbrains_plugin/platform.zip + key: chat-focus-platform-${{ hashFiles('jetbrains_plugin/gradle.properties', 'jetbrains_plugin/genPlatform.gradle') }} + - name: Build the full plugin without publishing + run: ./scripts/build.sh --mode release --clean --verbose + - uses: actions/upload-artifact@v4 + with: + name: chat-focus-plugin + path: jetbrains_plugin/build/distributions/*.zip + if-no-files-found: error + + windows-e2e: + name: Native typing and streaming chat (IDEA ${{ matrix.ide }}) + needs: build + runs-on: windows-2022 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - ide: "2025.3" + sha256: "6f7234b57444371c3af02ab31a15f76dd1d6e5e32eaabc96722777a9addcd023" + - ide: "2026.2.1" + sha256: "8aa16db629952300ab74d2d5e095150d1a534593e65bbbbd795b859759c00cbc" + defaults: + run: + shell: pwsh + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Validate fixture HTTP behavior on Windows + run: python -m unittest discover -s jetbrains_plugin/scripts/ui-harness -p test_streaming_fixture.py -v + - uses: actions/download-artifact@v4 + with: + name: chat-focus-plugin + path: focus-plugin + - name: Download pinned Windows IDE with bundled JBR and JCEF + env: + IDE_VERSION: ${{ matrix.ide }} + IDE_SHA256: ${{ matrix.sha256 }} + run: | + $ErrorActionPreference = 'Stop' + New-Item -ItemType Directory -Force focus-evidence | Out-Null + $archive = "idea-$env:IDE_VERSION.win.zip" + $url = "https://download.jetbrains.com/idea/$archive" + # Pinned from JetBrains' .sha256, not resolved from a latest feed. + $expected = $env:IDE_SHA256 + if ($expected -notmatch '^[a-fA-F0-9]{64}$') { throw 'Invalid vendor checksum' } + Invoke-WebRequest $url -OutFile "$env:RUNNER_TEMP/idea.zip" + $actual = (Get-FileHash "$env:RUNNER_TEMP/idea.zip" -Algorithm SHA256).Hash + if ($actual -ne $expected) { throw 'IDE archive checksum mismatch' } + "$url SHA256=$actual" | Set-Content focus-evidence/ide-download.txt + Expand-Archive "$env:RUNNER_TEMP/idea.zip" "$env:RUNNER_TEMP/focus-ide" + Remove-Item "$env:RUNNER_TEMP/idea.zip" + $launchers = @(Get-ChildItem "$env:RUNNER_TEMP/focus-ide" -Filter idea64.exe -Recurse) + if ($launchers.Count -ne 1) { throw 'Expected exactly one native IDE launcher' } + $ideDir = $launchers[0].Directory.Parent.FullName + "FOCUS_IDE_DIR=$ideDir" >> $env:GITHUB_ENV + - name: Run native desktop preflight and two-turn streaming focus test + run: | + $plugins = @(Get-ChildItem focus-plugin -Filter *.zip) + if ($plugins.Count -ne 1) { throw 'Expected exactly one plugin distribution' } + & ./jetbrains_plugin/scripts/ui-harness/run-windows.ps1 ` + -IdeDir $env:FOCUS_IDE_DIR -PluginZip $plugins[0].FullName ` + -EvidenceDir "$env:GITHUB_WORKSPACE/focus-evidence" + - name: Upload focus traces, screenshots, and run identity + if: always() + uses: actions/upload-artifact@v4 + with: + name: windows-chat-focus-${{ matrix.ide }}-${{ github.sha }} + path: focus-evidence/ + if-no-files-found: error + + live-solheim: + name: Live Solheim terminal and diff focus (WebStorm 2026.2.1) + needs: [build, windows-e2e] + if: >- + (github.event_name == 'workflow_dispatch' && inputs.live_solheim) || + (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'live-solheim')) + environment: solheim + runs-on: ${{ inputs.live_runner == 'windows11-x64' && fromJSON('["self-hosted", "Windows", "X64", "windows11-x64"]') || 'windows-2022' }} + timeout-minutes: 60 + defaults: + run: + shell: pwsh + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - uses: actions/download-artifact@v4 + with: + name: chat-focus-plugin + path: focus-plugin + - name: Download WebStorm 2026.2.1 with bundled JBR and JCEF + run: | + $ErrorActionPreference = 'Stop' + New-Item -ItemType Directory -Force live-evidence | Out-Null + $url = 'https://download.jetbrains.com/webstorm/WebStorm-2026.2.1.win.zip' + $expected = '08c65a4c2fbc5c8ede2d95de83fddd1665327c970f52c750e6e6414265f834d7' + Invoke-WebRequest $url -OutFile "$env:RUNNER_TEMP/webstorm-focus.zip" + $actual = (Get-FileHash "$env:RUNNER_TEMP/webstorm-focus.zip" -Algorithm SHA256).Hash + if ($actual -ne $expected) { throw 'WebStorm archive checksum mismatch' } + "$url SHA256=$actual" | Set-Content live-evidence/ide-download.txt + $idePath = Join-Path $env:RUNNER_TEMP ('webstorm-focus-' + [guid]::NewGuid().ToString('N')) + Expand-Archive "$env:RUNNER_TEMP/webstorm-focus.zip" $idePath + Remove-Item "$env:RUNNER_TEMP/webstorm-focus.zip" + $launchers = @(Get-ChildItem $idePath -Filter webstorm64.exe -Recurse) + if ($launchers.Count -ne 1) { throw 'Expected exactly one WebStorm launcher' } + "FOCUS_IDE_DIR=$($launchers[0].Directory.Parent.FullName)" >> $env:GITHUB_ENV + - name: Run three fresh live conversations without retrying failures + env: + SOLHEIM_API_KEY: ${{ secrets.SOLHEIM_API_KEY }} + REQUIRE_WINDOWS11: ${{ inputs.live_runner == 'windows11-x64' }} + run: | + $plugins = @(Get-ChildItem focus-plugin -Filter *.zip) + if ($plugins.Count -ne 1) { throw 'Expected exactly one plugin distribution' } + # The runner removes the key from its child-process environment. + # Retain it only in this PowerShell process for the next fresh profile. + $liveKey = $env:SOLHEIM_API_KEY + try { + for ($iteration = 1; $iteration -le 3; $iteration++) { + $env:SOLHEIM_API_KEY = $liveKey + & ./jetbrains_plugin/scripts/ui-harness/run-windows.ps1 ` + -IdeDir $env:FOCUS_IDE_DIR -PluginZip $plugins[0].FullName ` + -Provider solheim -DeadlineSeconds 780 -RequireWindows11:($env:REQUIRE_WINDOWS11 -eq 'true') ` + -EvidenceDir "$env:GITHUB_WORKSPACE/live-evidence/run-$iteration" + } + } finally { $liveKey = $null; $env:SOLHEIM_API_KEY = $null } + - name: Upload live focus evidence and tool coverage + if: always() + uses: actions/upload-artifact@v4 + with: + name: live-solheim-webstorm-2026.2.1-${{ github.sha }} + path: live-evidence/ + if-no-files-found: error + - name: Remove downloaded IDE on self-hosted runners + if: always() && inputs.live_runner == 'windows11-x64' + run: | + if ($env:FOCUS_IDE_DIR -and (Test-Path $env:FOCUS_IDE_DIR)) { + Remove-Item $env:FOCUS_IDE_DIR -Recurse -Force + } diff --git a/docs/KNOWN_ISSUES.md b/docs/KNOWN_ISSUES.md index 21021d9..b47fbdc 100644 --- a/docs/KNOWN_ISSUES.md +++ b/docs/KNOWN_ISSUES.md @@ -171,9 +171,8 @@ The plugin requires Node.js command line tools to function properly. Node.js exe ## 3. Focus Behavior Measured triggers, fixes, and remaining limits are recorded in -[the validation report](validation/chat-focus-preservation.md). Raw evidence -paths in that report are local, ignored build artifacts; the report includes -the relevant trace excerpts for readers of a clean checkout. +[the validation report](validation/chat-focus-preservation.md), with test setup +and evidence-review instructions in the linked harness README. ### 3.1 Terminal Creation Focus Restoration @@ -207,21 +206,15 @@ automatically activate chat. ### 3.3 Intermittent First-Turn Draft Character Loss -Open. Not fixed. Not conclusively attributed. Real provider-backed -conversations intermittently lose one draft character near the first -file-editor or diff opening. The latest run completed both turns but failed -draft preservation: 32/33 characters in the first turn, 6/6 in the follow-up. -The IDE trace recorded the key release without a matching press, with no -browser remount and the Swing owner stable on the chat OSR canvas. The -initiating platform call remains unproven. The same test stream found and fixed -the separate diff `preserveFocus` decoding bug; the live diff now opens with -`preserveFocus=true` and chat focus retained. A pure JDK probe -(`scripts/ui-harness/KeyDropProbe.java`) reproduced release-without-press only -under deliberate focus churn (4/120; control 0/120); it is analogous evidence, -not an IDE reproduction. A later passing run does not clear this issue. -Observed only in the Linux Xvfb/Openbox fixture on IU-253.28294.334. Other -platforms, builds, and live click races are untested. Trace excerpts and -failed-run references are in [the validation report](validation/chat-focus-preservation.md). +Open pending reporter or Windows 11 confirmation. The fix passed in hosted CI, +but the linked run predates the latest workflow, harness, Kotlin, and relay +changes, which require fresh CI validation. An earlier Windows live run lost one character during a preserving normal-file +open before the diff began. Preserving file and diff opens now share the +suspending file-editor API to avoid the synchronous loading loop. The expanded +regression opens 12 Java files followed by diffs while typing. The expanded +local regression, both Windows fixture jobs, and three fresh Windows live Solheim +conversations passed. The reporter's Windows 11 configuration remains unverified. See +[the validation report](validation/chat-focus-preservation.md). ## 4. General Troubleshooting Steps diff --git a/docs/validation/chat-focus-preservation.md b/docs/validation/chat-focus-preservation.md index 606eaf2..2befbb7 100644 --- a/docs/validation/chat-focus-preservation.md +++ b/docs/validation/chat-focus-preservation.md @@ -1,230 +1,95 @@ -# Chat focus validation summary - -Status 2026-10-17. Two plugin-owned focus defects are fixed and test-covered: -terminal creation stealing chat focus, and `preserveFocus` dropped by the diff -RPC decoder. The intermittent first-turn draft loss during real conversations -is **open**: not fixed, and not conclusively attributed. `docs/KNOWN_ISSUES.md` -links this file. Raw evidence paths below are relative to -`jetbrains_plugin/build/ui-harness/evidence/`. Those directories are local, -ignored artifacts kept in the investigation workspace; this file carries the -decisive excerpts for a clean checkout. - -## Plugin-owned fix 1: bounded terminal-creation focus restoration - -Proven trigger. `TerminalInstance.addToTerminalToolWindow` calls -`TerminalToolWindowManager.newTab`, which inserts and selects the new content -with focus enabled. The platform then moves Swing focus from the chat -`JBCefOsrComponent` to `JBTerminalPanel`, and typed text goes to the terminal. -The bytecode pin from the actual IDE (IU-253.28294.334) is in -`terminal-stack-baseline/terminal-manager-bytecode.txt`: - -```text -newTab: iconst_1; iconst_1; invokevirtual createNewTab(..., boolean, boolean) -createNewTab: ToolWindow.activate(runnable, true, true) -selection runnable: ContentManager.setSelectedContent(content, requestFocus) -``` - -The transition trace in `terminal-stack-baseline/progress.log` records T7 -production creation before the fix. The intended post-creation text was -`createpost`; none reached chat. - -```text -BATTERY-MARK ts=40771 step=T7-production-preserving-show -FOCUS-TIME ts=40796 prop=focusOwner old=com.intellij.ui.jcef.JBCefOsrComponent new=null -FOCUS-TIME ts=40797 prop=focusOwner old=null new=com.intellij.terminal.JBTerminalPanel -RECORD T7_creationFocusOwner = com.intellij.terminal.JBTerminalPanel -``` - -Fix. `TerminalCreationFocusGuard` captures the exact Swing chat owner before -terminal initialization. After creation it joins `ToolWindowManager.invokeLater` -and waits once for native focus to settle. It restores the captured component -once, only if focus ended inside the newly created terminal, chat stays visible, -the project is alive, and no user click, key, or Tab traversal intervened. -Focus in an editor or balloon is left alone. Input observation ends at the -callback or 1500 ms after creation. There is no polling loop, permanent -listener, repeated request, or unconditional restoration. Manager shutdown uses -a volatile flag, and refused JCEF registrations are disposed on the EDT outside -the ownership lock. The deliberate-show `ManualFocusGuard` is unchanged except -for documenting this bounded exception. - -Ordering evidence. `terminal-inline-callback-diagnostic/idea.log` records the -focus-settled callback before platform activation seeing the chat owner and -returning `restore=false`. `terminal-queued-callback-diagnostic/idea.log` -records the callback after the tool-window queue seeing `JBTerminalPanel` and -returning `restore=true` for T2 and T7, with exact typing probes passing. Later -attempts of that diagnostic died to system OOM -(`terminal-final-oom/`, `terminal-reduced-heap-oom/`); T7 observations after a -renderer kill are invalid, not regressions or passes. Final runs execute the -batteries sequentially and retain intermediate failures. - -## Plugin-owned fix 2: diff preserveFocus decoding - -The bundled extension passes `{preserveFocus:true}` to `vscode.diff`, but the -extension host wraps the options in a tuple -(`deps/vscode/src/vs/workbench/api/common/extHostApiCommands.ts:459`: -`[ViewColumn.from(...), TextEditorOpenOptions.from(...)]`). The Kotlin decoder -accepted only a map, so the real task logged `Opening diff, preserveFocus=false` -and the focus recorder caught an editor transition at ts=113909 with return to -the chat canvas at ts=113974. `decodeDiffPreserveFocus` now unwraps the tuple's -options member and still accepts direct-map callers. Missing or malformed flags -keep the existing false default. Four unit cases pin true, false, missing -options, and malformed input. - -Bundled extension note. The bundled Zoo Code extension 3.84.0 command runner -calls `terminal.show(true)` before `runCommand` (bundle SHA-256 -`2fe6fbde2ba7bc67e0c2e3f7eed62246da3cd9554a1ae2b9090faa1a7a0385b7`), so the -changed omitted-argument default does not force terminal focus on that path. -Explicit false or an omitted argument from another caller still requests -terminal focus. - -## Test status - -The CI allowlist command in `.github/workflows/pr.yml` passes with **104 tests, -0 failures, 0 errors, 0 skipped** (local output `review-3-unit-tests.log`). -Historical 86 and 100 counts describe earlier builds. The local relay tests -pass (`python3 -m unittest test_solheim_proxy.py`; twelve cases after the -run-4 relay hardening). Shell syntax and diff whitespace checks pass. The full -platform-fixture suite was not rerun; its previously documented failures are -not claimed fixed. - -## Harness results - -Live-run environment: harness IDE IU-253.28294.334, Linux Xvfb with Openbox, -heap 768 MB, 4 CPUs, plugin zip SHA-256 -`4d9b34ca46c4bbe0aba7637181e697da4f1bcbb96590d87efdf99c72ab3865e4` unless -noted. Conversation runs use the real bundled extension, the real React -textarea, real tool execution, and Solheim `qwen3.8-27b` through the -loopback-only relay (`scripts/ui-harness/solheim_proxy.py`). No simulated -provider responses or injected chat input are used for the conversation battery. - -Corrected runs on plugin-owned paths: - -- Mode F: PASS. Typing reaches the input with the OSR canvas focused - (`typedWithCanvasFocusVerdict=true`) and stops when Swing focus moves - elsewhere (`keysLostWhenFocusMovedVerdict=true`). Evidence: `f/`. -- Mode T: plugin-owned paths pass. T2 production creation, T2R streaming and - editor replay, T3 visibility-only show, T7 creation and preserving show, and - T8 deliberate show. Evidence: `t/`. Stock first-show (T1) and stock - new-tab/selection (T4) still focus the terminal; see the classification - below. -- Mode G, corrected final run: PASS. G6 manual click recovery, G7 preserving - editor bridge with exact owner identity and post-bridge typing, and G8 - disposal typing. G1 reload and G5/G6 raw hide-show without manual refocus - still do not type; recorded as-is. Evidence: `g/`. The earlier invalid run - that clicked the retired browser is `g-native-stale-browser/`. -- Mode P, final instrumented run: PASS in 103 seconds. Thirty exact first-turn - samples and five exact follow-up samples; history grew 21 then 29 messages; - three real commands exited 0; the diff logged `preserveFocus=true`. Evidence: - `p/`. This 35/35 pass is historical; the latest conversation run below - supersedes it. - -Deliberate stock user actions, classified separately. The first Terminal -tool-window show and stock tab-selection and new-session paths move focus to -the terminal by platform design. The plugin does not wrap these stock actions, -and this report does not classify them as defects. The preserving-show and -deliberate-show harness checks for these paths remain required. - -Fixture corrections, not product issues. The IDEA script-launcher warning -balloon covered the textarea in early runs; the fixture now uses the native -launcher. One G prerequisite clicked the browser retired during webview -replacement; the harness now references the mounted browser. The driver also -waits for smart mode and a showing browser before its setup click. These were -test-setup defects. No balloon-restoration or permanent focus code was added to -the plugin. - -## Latest real provider run: FAIL - -**FAIL-REAL-CONVERSATION**, 123 seconds, on the same executable plugin -artifact. Both real turns completed; history grew, commands ran, the fixture -changed, and the diff opened with `preserveFocus=true`. Draft preservation -failed: the first turn kept **32 of 33** characters, the follow-up kept **6 of -6**. The first mismatch is sample 16 (`snapshot=19`): 15 actual characters -against 16 expected, with the chat still active. Raw evidence: -`review-3-standalone-p/`; earlier startup failures: -`review-3-standalone-startup-{1,2,3}/`. - -No browser remount occurred around the loss: the only mount and repeated-mount -logs are at 01:05:59 and 01:06:00, and the diff opened at 01:06:46.305. The -Swing owner stayed on the chat OSR canvas. The affected sample logs only the -key release in the IDE PRE/POST trace: - -```text -E2EAWT PRE ts=1791940006381 when=1791940006326 id=402 key=88 char=120 consumed=false owner=com.intellij.ui.jcef.JBCefOsrComponent@5f8aec10 -E2EAWT POST ts=1791940006382 when=1791940006326 id=402 key=88 consumed=false source=com.intellij.ui.jcef.JBCefOsrComponent@5f8aec10 -RECORD P_expected_16 = xxxxxxxxxxxxxxxx -``` - -No matching key press or DOM keydown or input arrived for that sample. The -trace narrows the gap to delivery before the recorded IDE preprocessors. It -does not prove which platform call caused it. - -Two earlier real runs on this build each lost one character near the first -file-editor opening (`p-tuple-fix-one-key-loss/`, `p-dom-key-loss/`); DOM -tracing there showed no textarea replacement and no blur. An intermediate -complete run is preserved as FAIL for a stale probe sample -(`p-complete-stale-probe/`), and a partial first attempt as -`p-native-partial-first/`. - -## JDK probe: analogous evidence and its limits - -`jetbrains_plugin/scripts/ui-harness/KeyDropProbe.java` is a JDK-only probe -with no IDE, plugin, or JCEF code. It types with `java.awt.Robot` and counts -press, typed, and release deliveries. In a local manual run, deliberate focus -churn produced release-without-press **4 times in 120 keys**, while the control -mode produced **0 in 120**. The probe's churn styles change the focus owner; -the IDE trace above kept the owner stable. The probe therefore shows that a -release can arrive without its press under focus churn in a pure JDK and Swing -stack. It does not reproduce the IDE mechanism, does not identify the -initiating IDE platform call, and does not prove a JDK root cause or clear the -product defect. Single runs are nondeterministic; compare control and churn on -the same host in the same session. - -## Remaining validation gaps - -- The intermittent first-turn draft loss is **open**: no fix, no proven - initiating call, and later passing runs do not clear it. -- macOS, native Windows, and other IDE builds are untested. The live - preserving-diff chain is exercised on build 253 only; builds 233/263 are - untested. -- Live user-click races during the guard window are untested. Controlled guard - lifecycle tests cover clicks, Tab traversal, expiry, repeated callbacks, - hidden chat, and disposal before or while queued. -- The full platform-fixture suite was not rerun; its previously documented - failures are not claimed fixed. - -## Reproduction - -Build and harness modes T/G/F/P: - -```sh -cd jetbrains_plugin -./gradlew buildPlugin -PdebugMode=release -PvscodePlugin=zoo-code --no-configuration-cache -cd build/ui-harness -unzip -oq ../distributions/ZooCode-3.78.0.zip -d plugins-dist -HARNESS_HEAP_MB=768 HARNESS_CPU_COUNT=4 IDE_DIR=$PWD/ide-253 DEADLINE_SECONDS=600 bash bin/run-test.sh t -HARNESS_HEAP_MB=768 HARNESS_CPU_COUNT=4 IDE_DIR=$PWD/ide-253 DEADLINE_SECONDS=600 bash bin/run-test.sh g -HARNESS_HEAP_MB=768 HARNESS_CPU_COUNT=4 IDE_DIR=$PWD/ide-253 DEADLINE_SECONDS=600 bash bin/run-test.sh f -``` - -The live provider conversation needs the relay and key setup from -`jetbrains_plugin/scripts/ui-harness/README.md`. The JDK probe needs only Xvfb -and a headful JDK: - -```sh -cd jetbrains_plugin/scripts/ui-harness -xvfb-run -a /path/to/headful-jdk/bin/java KeyDropProbe.java control 120 7 -xvfb-run -a /path/to/headful-jdk/bin/java KeyDropProbe.java churn 120 7 hidden-frame -``` - -## Evidence index (local, ignored) - -Under `jetbrains_plugin/build/ui-harness/evidence/`: `review-3-standalone-p/` -(latest FAIL), `review-3-standalone-startup-{1,2,3}/`, `p/` (historical -instrumented pass), `p-tuple-fix-one-key-loss/`, `p-dom-key-loss/`, -`p-complete-stale-probe/`, `p-native-partial-first/`, -`terminal-stack-baseline/`, `terminal-replay-baseline/`, -`terminal-inline-callback-diagnostic/`, -`terminal-queued-callback-diagnostic/`, `terminal-final-oom/`, -`terminal-reduced-heap-oom/`, `t/`, `g/`, `f/`, `g-native-stale-browser/`, -unit XML in `review-fixes-unit-results/` and -`terminal-creation-unit-results/`, and earlier reports `FOCUS_FIX_REPORT*.md`. +# Chat focus validation + +Updated 2026-11-08. [Issue #5](https://github.com/Zoo-Code-Org/Zoo-Code-Jetbrains/issues/5) +has a file and diff loading fix that passed in hosted CI. The linked run predates +the latest workflow, harness, Kotlin, and relay changes, which require a fresh +Windows run. Reporter or Windows 11 confirmation remains required before closing #5. + +## Current finding and fix + +The [Windows live run](https://github.com/Zoo-Code-Org/Zoo-Code-Jetbrains/actions/runs/42384965962/job/128442585155) +with the diff-only fix lost one first-turn character: 28/29 survived, followed by +7/7 in the second turn. Both turns completed, all three commands exited 0, the +file edit succeeded, and sampled chat focus stayed intact. + +The missing key falls inside `showTextDocument(..., preserveFocus=true)`, before +the diff call begins. Normal file opens still used the synchronous editor API, +whose loading loop can consume input before normal keyboard dispatch. Preserving +file and diff opens now share the suspending API with focus disabled. Options +use the stable JVM no-argument constructor for IDE binary compatibility. + +The regression now exercises 12 normal Java file opens, each followed by a diff, +through the production APIs while typing. Every file and diff tab must open. +The broader fix passed locally on WebStorm 2026.2.1 with the CI extension/runtime +held constant: all 12 files and 12 diffs opened, both streamed turns completed, +and all 23 first-turn and 11 follow-up characters survived. Live Solheim also +passed with 21/21 and 5/5 characters, all three commands exiting 0, the file edit +and preserving diff complete, and all nine provider responses HTTP 200 +(`https://api.solheim.ai/v1`, `qwen3.8-27b`). The earlier diff-only runs did not +establish that the normal-file path was safe. Reproduction is intermittent, so a single green run +is insufficient evidence to close the issue. + +The [latest hosted run](https://github.com/Zoo-Code-Org/Zoo-Code-Jetbrains/actions/runs/42531095791/job/128887638578) +passed the build, both IntelliJ fixture jobs, and the first live conversation. +The second live conversation stopped at `mistake_limit_reached` after repeated +`update_todo_list` calls. All 108 first-turn characters survived and sampled +focus stayed intact, but required commands and the follow-up were incomplete. +This is a failed live validation, not a passing result for the latest code. +The harness now discourages unrelated todo tools and reports this stop promptly; +these adjustments still need Windows validation. + +## Existing fixes + +- `TerminalCreationFocusGuard` restores the captured chat component once when + plugin-owned terminal creation takes focus, provided no user interaction or + unrelated focus change intervenes. The guard is bounded and test-covered. +- Diff RPC decoding preserves `preserveFocus=true` through the extension host's + tuple-shaped options. This prevents an unintended editor-focus request. + +Deliberate user actions that focus the terminal are expected behavior. + +## Test coverage + +Recorded runs use the bundled Zoo Code extension 3.84.0. The Windows results below +predate the latest workflow, harness, Kotlin, and relay changes; those changes +require fresh CI validation. + +| Environment | Scenario | Result | +| --- | --- | --- | +| Windows Server 2022, IntelliJ 2025.3 and 2026.2.1 | Streaming plus 12 file/diff sequences | Passed in [CI](https://github.com/Zoo-Code-Org/Zoo-Code-Jetbrains/actions/runs/42409470007) | +| Windows Server 2022, WebStorm 2026.2.1 | Three fresh live Solheim conversations | All passed in [CI](https://github.com/Zoo-Code-Org/Zoo-Code-Jetbrains/actions/runs/42409470007/job/128516604245) | +| Linux, IntelliJ 2025.3 | Live Solheim commands and diff | Passed; does not clear intermittent failures | +| Linux, WebStorm 2026.2.1 | Streaming plus 12 file/diff sequences | Passed with broader fix | +| Linux, WebStorm 2026.2.1 | Live Solheim commands and diff | Passed with broader fix | + +The harness uses native keyboard events in the real React/JCEF input. It requires +exact drafts, sampled input/window focus, streamed history updates, and completed +turns. The follow-up prompt is checked before submission. Live Windows runs also +require terminal completion markers, required provider tool calls, the edited +fixture, and preserving-diff log evidence. Setup failures, timeouts, and missing +coverage fail the test; failed typing samples are never repaired or discarded. + +Windows fixture jobs now exercise preserving files followed by diffs through +the production APIs and verify each tab opened. Files are prepared before typing to separate +filesystem discovery from editor loading. Live Solheim adds terminal and real +model/tool activity; fixture results do not validate those paths. + +## Running and reviewing tests + +See the [harness README](../../jetbrains_plugin/scripts/ui-harness/README.md) for +local setup, license/trial prerequisites, and commands. The +[Windows workflow](../../.github/workflows/windows-chat-focus.yml) runs fixture +checks on PRs. Live runs require the `live-solheim` label or manual opt-in, plus +approval of the protected `solheim` environment with `SOLHEIM_API_KEY` configured. + +Review screenshots, `evidence.json`, focus/key traces, provider metadata, IDE logs, +and `live-validation.json` in workflow artifacts. Confirm the tested commit, IDE +build, OS, and plugin digest, and verify that required tools actually ran before +accepting a pass. Startup dialogs and covered input targets are setup failures, +not evidence of a streaming focus regression. + +The linked hosted CI run passed both Windows fixture jobs and three fresh live Solheim +conversations, including native typing in both turns, terminal commands, edits, +and preserving diffs. Hosted Windows Server evidence supports the fix. Close #5 +only after the reporter or a Windows 11 run confirms it. Use an interactive +Windows 11 x64 runner or a manual reproduction for that configuration. diff --git a/jetbrains_plugin/scripts/ui-harness/DesktopTypingProbe.java b/jetbrains_plugin/scripts/ui-harness/DesktopTypingProbe.java new file mode 100644 index 0000000..0d17fbc --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/DesktopTypingProbe.java @@ -0,0 +1,52 @@ +import java.awt.*; +import java.awt.event.KeyEvent; +import java.nio.file.*; +import javax.imageio.ImageIO; +import javax.swing.*; + +/** Fails before IDE startup if the runner cannot deliver real desktop keys. */ +public class DesktopTypingProbe { + public static void main(String[] args) throws Exception { + Path evidence = Paths.get(args[0]); + Files.createDirectories(evidence); + JFrame frame = new JFrame("Zoo native desktop typing preflight"); + JTextArea input = new JTextArea(); + try { + SwingUtilities.invokeAndWait(() -> { + frame.add(input); + frame.setSize(600, 200); + frame.setLocationRelativeTo(null); + frame.setAlwaysOnTop(true); + frame.setVisible(true); + }); + Robot robot = new Robot(); + robot.setAutoDelay(100); + Thread.sleep(1000); + Point[] point = {null}; + SwingUtilities.invokeAndWait(() -> point[0] = input.getLocationOnScreen()); + robot.mouseMove(point[0].x + 40, point[0].y + 40); + robot.mousePress(java.awt.event.InputEvent.BUTTON1_DOWN_MASK); + robot.mouseRelease(java.awt.event.InputEvent.BUTTON1_DOWN_MASK); + for (char c : "desktop".toCharArray()) { + int key = KeyEvent.getExtendedKeyCodeForChar(c); + robot.keyPress(key); + robot.keyRelease(key); + } + Thread.sleep(500); + ImageIO.write(robot.createScreenCapture(new Rectangle(Toolkit.getDefaultToolkit().getScreenSize())), + "png", evidence.resolve("desktop-preflight.png").toFile()); + String[] actual = {null}; + boolean[] focused = {false}; + SwingUtilities.invokeAndWait(() -> { + actual[0] = input.getText(); + focused[0] = input.isFocusOwner(); + }); + boolean pass = "desktop".equals(actual[0]) && focused[0]; + Files.writeString(evidence.resolve("desktop-preflight.txt"), + "pass=" + pass + " text=" + actual[0] + " focused=" + focused[0]); + if (!pass) throw new IllegalStateException("Native desktop typing unavailable"); + } finally { + SwingUtilities.invokeAndWait(frame::dispose); + } + } +} diff --git a/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java b/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java index 49d774e..831aa4c 100644 --- a/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java +++ b/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java @@ -46,11 +46,20 @@ public final class HarnessDriver implements AppLifecycleListener { private static final java.util.regex.Pattern FIXTURE_COMMAND = java.util.regex.Pattern.compile( "printf SOLHEIM_(FIRST|SECOND|THIRD)(; sleep [0-9]+)?(; cat src/Main\\.java)?"); private long lastApprovedTs = -1; + private final java.util.Set approvedCommands = new java.util.HashSet<>(); + private final boolean fixtureMode = Boolean.getBoolean("zoo.harness.fixture"); + private final boolean diffStress = Boolean.getBoolean("zoo.harness.diffStress"); + private final boolean windowsLive = !fixtureMode && System.getProperty("os.name", "").startsWith("Windows"); + + private String windowsCommand(String step) { + return "powershell.exe -NoProfile -File .focus-ci/" + step + ".ps1"; + } private final Object evidenceLock = new Object(); private Path evidenceDir; private Path progressLog; private List checks = new ArrayList<>(); + private volatile boolean startupComplete; @Override public void appFrameCreated(List commandLineArgs) { @@ -92,6 +101,18 @@ private void waitForProjectAndRun() { private void run(Project project, String mode) { log("driver started, mode=" + mode); try { + if (diffStress) { + Path dir = Paths.get(project.getBasePath(), ".focus-diffs"); + Files.createDirectories(dir); + for (int i=0; i<12; i++) { + Files.writeString(dir.resolve("before-" + i + ".java"), "class Before" + i + " {}\n"); + Files.writeString(dir.resolve("after-" + i + ".java"), "class After" + i + " {}\n"); + } + com.intellij.openapi.vfs.VirtualFile files = com.intellij.openapi.vfs.LocalFileSystem.getInstance() + .refreshAndFindFileByIoFile(dir.toFile()); + if (files == null) throw new IllegalStateException("Diff fixture directory unavailable"); + files.refresh(false, true); + } record("jcefState", reflectJcefSupport()); record("ideBuild", com.intellij.openapi.application.ApplicationInfo.getInstance().getBuild().asString()); record("javaVersion", System.getProperty("java.version")); @@ -134,9 +155,27 @@ private Class zooClass(String name) throws Exception { private boolean addTitleHandler(Object jbCefBrowser, Object cefBrowser, AtomicReference capturedTitle) { try { Object jbCefClient = invoke(jbCefBrowser, "getJBCefClient"); - Class handlerIfc = Class.forName("org.cef.handler.CefDisplayHandler"); + // Newer IDEs isolate JCEF from this test plugin's class loader. + // Use the interface exposed by the running client's API so the proxy + // has the exact type identity required by addDisplayHandler. + Method register = null; + for (Method method : jbCefClient.getClass().getMethods()) { + Class[] parameters = method.getParameterTypes(); + if (method.getName().equals("addDisplayHandler") && parameters.length == 2 && + parameters[0].getName().equals("org.cef.handler.CefDisplayHandler")) { + register = method; + break; + } + } + if (register == null) throw new IllegalStateException("JCEF addDisplayHandler API unavailable"); + Class handlerIfc = register.getParameterTypes()[0]; Object proxy = Proxy.newProxyInstance(handlerIfc.getClassLoader(), new Class[]{handlerIfc}, (p, m, a) -> { + if (m.getDeclaringClass() == Object.class) { + if (m.getName().equals("hashCode")) return System.identityHashCode(p); + if (m.getName().equals("equals")) return p == a[0]; + if (m.getName().equals("toString")) return "Zoo focus harness display handler"; + } if ("onTitleChange".equals(m.getName()) && a.length >= 2 && a[1] instanceof String) { capturedTitle.set((String) a[1]); if (((String)a[1]).startsWith("ZFE2E:")) { @@ -147,6 +186,10 @@ private boolean addTitleHandler(Object jbCefBrowser, Object cefBrowser, AtomicRe } if ("onConsoleMessage".equals(m.getName()) && a.length >= 4) { String msg = String.valueOf(a[2]); + if (msg.startsWith("ZFE2E:")) { + JsonObject reply=JsonParser.parseString(msg.substring(6)).getAsJsonObject(); + realReplies.put(reply.get("snapshot").getAsInt(),reply); + } log("PAGE-CONSOLE " + a[1] + " " + msg); if (msg.startsWith("ZFC:")) { consoleLines.add(msg); @@ -164,17 +207,8 @@ private boolean addTitleHandler(Object jbCefBrowser, Object cefBrowser, AtomicRe } return null; }); - for (Method m : jbCefClient.getClass().getMethods()) { - if (m.getName().equals("addDisplayHandler")) { - Class[] pt = m.getParameterTypes(); - if (pt.length == 2 && pt[0].isAssignableFrom(handlerIfc)) { - m.invoke(jbCefClient, proxy, cefBrowser); - return true; - } - } - } - LOG.warn("addDisplayHandler not found on " + jbCefClient.getClass()); - return false; + register.invoke(jbCefClient, proxy, cefBrowser); + return true; } catch (Throwable t) { LOG.warn("title handler attach failed", t); return false; @@ -214,11 +248,19 @@ private void mark(String step) { // Manual provider-backed battery: real React textarea, real extension tools. private JsonObject realSnapshot(Object webview, AtomicReference title) throws Exception { int id=++realSequence; - invoke(webview, "executeJavaScript", "document.title='ZFE2E:' + JSON.stringify((()=>{let e=[...document.querySelectorAll('textarea')].find(x=>x.offsetWidth&&x.offsetHeight);let r=e&&e.getBoundingClientRect();let z=window.__focusE2E||{};let m=z.messages||[];let last=m[m.length-1]||{};return {snapshot:"+id+",input:!!e,value:e?e.value:null,active:e===document.activeElement,disabled:e?e.disabled:null,rect:r?{x:r.x,y:r.y,w:r.width,h:r.height}:null,messages:m.length,events:z.events||0,partial:z.partial||0,lastAsk:last.ask||null,lastTs:last.ts||null,lastText:typeof last.text==='string'?last.text.slice(0,300):null,lastSay:last.say||null,lastPartial:!!last.partial,provider:z.provider||null,body:document.body.innerText.slice(-1200)};})());"); + String script="(()=>{const snapshot=JSON.stringify((()=>{let e=[...document.querySelectorAll('textarea')].find(x=>x.offsetWidth&&x.offsetHeight);let r=e&&e.getBoundingClientRect();let z=window.__focusE2E||{};let m=z.messages||[];let last=m[m.length-1]||{};return {snapshot:"+id+",input:!!e,value:e?e.value:null,active:e===document.activeElement,windowFocused:document.hasFocus(),disabled:e?e.disabled:null,rect:r?{x:r.x,y:r.y,w:r.width,h:r.height}:null,messages:m.length,events:z.events||0,partial:z.partial||0,lastAsk:last.ask||null,lastTs:last.ts||null,lastText:typeof last.text==='string'?last.text.slice(0,300):null,lastSay:last.say||null,lastPartial:!!last.partial,provider:z.provider||null,body:document.body.innerText.slice(-1200)};})());console.log('ZFE2E:'+snapshot);document.title='ZFE2E:'+snapshot;})();"; + runOnEdt(() -> invoke(webview, "executeJavaScript", script)); + long resend=System.currentTimeMillis()+1000; long deadline=System.currentTimeMillis()+30000; while(System.currentTimeMillis()= resend) { + // Observation only: no input or focus changes. A page title can + // be overwritten and JCEF may drop a queued evaluation. + runOnEdt(() -> invoke(webview, "executeJavaScript", script)); + resend=System.currentTimeMillis()+1000; + } Thread.sleep(50); } throw new IllegalStateException("real DOM snapshot unavailable; pageLoaded="+invoke(webview,"isPageLoaded")+" disposed="+invoke(webview,"isDisposed")); @@ -248,21 +290,51 @@ private void runRealConversation(Project project) throws Exception { }; runOnEdt(()->{com.intellij.ide.IdeEventQueue.getInstance().addPreprocessor(pre,project);com.intellij.ide.IdeEventQueue.getInstance().addPostprocessor(post,project);}); Object manager = project.getService(zooClass("org.zoocode.jetbrains.webview.WebViewManager")); - Object webview = poll(120000, () -> invoke(manager, "getLatestWebView")); - poll(120000, () -> invoke(webview, "isPageLoaded")); + // Startup can replace a webview. Select the loaded, mounted browser after + // indexing rather than waiting forever on an instance that was retired. + Object webview = poll(180000, () -> { + Object candidate = invoke(manager, "getLatestWebView"); + if (candidate == null || !Boolean.TRUE.equals(invoke(candidate, "isPageLoaded")) || + Boolean.TRUE.equals(invoke(candidate, "isDisposed")) || + com.intellij.openapi.project.DumbService.getInstance(project).isDumb()) return null; + final boolean[] showing = {false}; + runOnEdt(() -> { + showing[0] = ((Component) invoke(field(candidate, "browser"), "getComponent")).isShowing(); + for (Window window : Window.getWindows()) { + if (!window.isShowing()) continue; + String title = window instanceof Dialog ? ((Dialog) window).getTitle() + : window instanceof Frame ? ((Frame) window).getTitle() : ""; + if ((window instanceof Dialog && ((Dialog) window).isModal()) || "Manage Licenses".equals(title)) { + showing[0] = false; + } + } + }); + return showing[0] ? candidate : null; + }); + if (webview == null) throw new IllegalStateException("No loaded, mounted chat browser after indexing"); + startupComplete = true; Object browser = field(webview, "browser"); AtomicReference title = new AtomicReference<>(); - addTitleHandler(browser, invoke(browser, "getCefBrowser"), title); - invoke(webview, "executeJavaScript", "window.__focusE2E={messages:[],events:0,partial:0};window.addEventListener('message',e=>{let d=e.data,z=window.__focusE2E;if(!d)return;z.events++;if(d.type==='state'&&d.state){if(d.state.clineMessages)z.messages=d.state.clineMessages;z.provider=d.state.apiConfiguration&&d.state.apiConfiguration.apiProvider;}if(d.type==='partialMessage'){z.partial++;if(d.partialMessage){let i=z.messages.findIndex(m=>m.ts===d.partialMessage.ts);if(i>=0)z.messages[i]=d.partialMessage;else z.messages.push(d.partialMessage);}}});"); + if (!addTitleHandler(browser, invoke(browser, "getCefBrowser"), title)) { + throw new IllegalStateException("Cannot register JCEF snapshot handler; see IDE log for cause"); + } + invoke(webview, "executeJavaScript", "window.__focusE2E={messages:[],events:0,partial:0};window.addEventListener('message',e=>{let d=e.data,z=window.__focusE2E;if(!d)return;z.events++;if(d.type==='state'&&d.state){if(d.state.clineMessages)z.messages=d.state.clineMessages.slice();z.provider=d.state.apiConfiguration&&d.state.apiConfiguration.apiProvider;}if(d.type==='messageUpdated'||d.type==='partialMessage'){let m=d.clineMessage||d.partialMessage;if(m){if(m.partial)z.partial++;let i=z.messages.findIndex(x=>x.ts===m.ts);if(i>=0)z.messages[i]=m;else z.messages.push(m);}}});"); invoke(webview,"executeJavaScript","window.__focusNodeSeq=0;function e2eNode(e){if(!e)return null;if(!e.__e2eid)e.__e2eid=++window.__focusNodeSeq;return e.tagName+'@'+e.__e2eid;}for(let kind of ['keydown','input','focusin','focusout'])document.addEventListener(kind,e=>{if(kind==='keydown'&&!['x','y'].includes(e.key))return;console.info('E2EKEY:'+JSON.stringify({kind,ts:Date.now(),key:e.key||null,mod:e.ctrlKey||e.altKey||e.metaKey||false,node:e2eNode(e.target),active:e2eNode(document.activeElement),hasFocus:document.hasFocus(),length:typeof e.target.value==='string'?e.target.value.length:null}));},true);for(let kind of ['focus','blur'])window.addEventListener(kind,e=>console.info('E2EWINDOW:'+JSON.stringify({kind,ts:Date.now(),active:e2eNode(document.activeElement),hasFocus:document.hasFocus()})),true);"); realSend(webview, "{\"type\":\"webviewDidLaunch\"}"); realSend(webview, "{\"type\":\"mode\",\"text\":\"code\"}"); Thread.sleep(1500); String configuration = "{\"type\":\"upsertApiConfiguration\",\"text\":\"focus-solheim\",\"apiConfiguration\":{\"apiProvider\":\"openai\",\"openAiBaseUrl\":\"http://127.0.0.1:19487/v1\",\"openAiApiKey\":\"LOCAL_TOKEN\",\"openAiModelId\":\"qwen3.8-27b\",\"openAiStrictToolSchemas\":false,\"openAiExtraBody\":\"{\\\"chat_template_kwargs\\\":{\\\"enable_thinking\\\":false}}\",\"openAiCustomModelInfo\":{\"contextWindow\":131072,\"maxTokens\":4096,\"supportsImages\":false,\"supportsPromptCache\":false,\"supportsNativeTools\":true,\"inputPrice\":0,\"outputPrice\":0}}}"; JsonObject config = JsonParser.parseString(configuration).getAsJsonObject(); - config.getAsJsonObject("apiConfiguration").addProperty("openAiApiKey", System.getenv("SOLHEIM_RELAY_TOKEN")); + JsonObject apiConfig = config.getAsJsonObject("apiConfiguration"); + apiConfig.addProperty("openAiApiKey", fixtureMode ? "focus-fixture-local-only" : System.getenv("SOLHEIM_RELAY_TOKEN")); + if (fixtureMode) { + apiConfig.addProperty("openAiModelId", "focus-fixture"); + apiConfig.remove("openAiExtraBody"); + } + if (fixtureMode) config.addProperty("text", "focus-fixture"); realSend(webview, config.toString()); realSend(webview, "{\"type\":\"updateSettings\",\"updatedSettings\":{\"autoApprovalEnabled\":true,\"alwaysAllowReadOnly\":true,\"alwaysAllowWrite\":true,\"alwaysAllowExecute\":true,\"allowedCommands\":[\"printf\",\"sleep\",\"cat\",\"pwd\"],\"terminalShellIntegrationDisabled\":false,\"destructiveCommandGuardEnabled\":false,\"maxRequests\":20,\"telemetrySetting\":\"disabled\"}}"); + if (!fixtureMode) realSend(webview, "{\"type\":\"updateSettings\",\"updatedSettings\":{\"alwaysAllowExecute\":false}}"); realSend(webview, "{\"type\":\"didShowAnnouncement\"}"); invoke(webview, "executeJavaScript", "[...document.querySelectorAll('button')].find(b=>b.textContent.trim()==='Close')?.click();"); JsonObject snap = null; @@ -277,11 +349,44 @@ private void runRealConversation(Project project) throws Exception { return showing[0] && !com.intellij.openapi.project.DumbService.getInstance(project).isDumb(); }); if (!Boolean.TRUE.equals(ready)) throw new IllegalStateException("Chat remained hidden during indexing"); + snap = realSnapshot(webview,title); screenshot("p-configured"); - clickInBrowser(browser, snap.getAsJsonObject("rect")); + // Establish native focus before typing anything. A freshly mounted OSR + // surface may paint after its DOM is ready. Setup clicks are bounded; + // no clicks or focus repairs occur inside either draft typing loop. + boolean initialFocus = false; + for (int attempt=1; attempt<=10; attempt++) { + snap=realSnapshot(webview,title); + clickInBrowser(browser, snap.getAsJsonObject("rect")); + Thread.sleep(500); + snap=realSnapshot(webview,title); + if (snap.get("active").getAsBoolean() && snap.get("windowFocused").getAsBoolean()) { + record("P_initialFocusClicks",attempt); + initialFocus=true; + break; + } + } + if (!initialFocus) { fail("nativeChatFocusUnavailable", "Setup clicks did not focus the mounted chat input"); return; } + { + typeText("smoke"); + Thread.sleep(500); + JsonObject smoke = realSnapshot(webview, title); + record("P_nativeChatTypingSmoke", smoke); + if (!"smoke".equals(smoke.get("value").getAsString()) || + !smoke.get("active").getAsBoolean() || !smoke.get("windowFocused").getAsBoolean()) { + fail("nativeChatTypingUnavailable", "Native keys did not reach the focused React textarea"); + return; + } + Robot clear = new Robot(); + clear.keyPress(KeyEvent.VK_CONTROL); clear.keyPress(KeyEvent.VK_A); + clear.keyRelease(KeyEvent.VK_A); clear.keyRelease(KeyEvent.VK_CONTROL); + clear.keyPress(KeyEvent.VK_BACK_SPACE); clear.keyRelease(KeyEvent.VK_BACK_SPACE); + Thread.sleep(300); + } Toolkit.getDefaultToolkit().getSystemClipboard().setContents( - new java.awt.datatransfer.StringSelection("Use tools in this throwaway workspace: execute_command printf SOLHEIM_FIRST; sleep 5. Read src/Main.java, then apply_diff replacing throwaway with solheim_e2e. Execute printf SOLHEIM_SECOND; sleep 5; cat src/Main.java. No files outside this workspace. Briefly explain each step and finish with attempt_completion. Do not ask questions."),null); + new java.awt.datatransfer.StringSelection(fixtureMode ? "Stream the fixture response and finish with attempt_completion." : windowsLive ? "Use tools in this throwaway Windows workspace. Run execute_command with exactly: " + windowsCommand("first") + ". Read src/Main.java, then use apply_diff to replace throwaway with solheim_e2e. Run execute_command with exactly: " + windowsCommand("second") + ". Do not edit .focus-ci or files outside this workspace. Use only execute_command, read_file, apply_diff, and attempt_completion for this task. Do not create or update a todo list. Briefly explain each step and finish with attempt_completion. Do not ask questions." : "Use tools in this throwaway workspace: execute_command printf SOLHEIM_FIRST; sleep 5. Read src/Main.java, then apply_diff replacing throwaway with solheim_e2e. Execute printf SOLHEIM_SECOND; sleep 5; cat src/Main.java. No files outside this workspace. Use only execute_command, read_file, apply_diff, and attempt_completion for this task. Do not create or update a todo list. Briefly explain each step and finish with attempt_completion. Do not ask questions."),null); Robot robot = new Robot(); + robot.setAutoDelay(90); robot.keyPress(KeyEvent.VK_CONTROL); robot.keyPress(KeyEvent.VK_V); robot.keyRelease(KeyEvent.VK_V); robot.keyRelease(KeyEvent.VK_CONTROL); Thread.sleep(1000); @@ -294,43 +399,77 @@ private void runRealConversation(Project project) throws Exception { if (snap.get("messages").getAsInt()==0) { record("P_afterSubmit",snap); fail("realTaskMissing","No task appeared after submitting prompt");return; } // Establish a single focus point before tools run; subsequent sampling never refocuses. clickInBrowser(browser,snap.getAsJsonObject("rect")); + record("P_beforeDraftTyping",realSnapshot(webview,title)); String expected=""; boolean typingHealthy=true; + boolean focusHealthy=true; + int initialPartial=snap.get("partial").getAsInt(); boolean completed=false; + java.util.concurrent.Future diffOpens = diffStress ? AppExecutorUtil.getAppExecutorService().submit(() -> { + try { + for (int i=0; i<12; i++) { + openPreservingEditors(project, i); + Thread.sleep(350); + } + } catch (Throwable e) { + StringWriter stack = new StringWriter(); + e.printStackTrace(new PrintWriter(stack)); + log("preserving editor sequence failed: " + stack); + throw new RuntimeException(e); + } + }) : null; int initialMessages=snap.get("messages").getAsInt(); long deadline=System.currentTimeMillis()+240000; int sample=0; while(System.currentTimeMillis()2) {completed=true;break;} if (!answerCommandAsk(webview, snap, ask)) return; } record("P_realTaskCompleted",completed); + if (diffOpens != null) { + diffOpens.get(60, TimeUnit.SECONDS); + record("P_preservingFilesCompleted",12); + record("P_preservingDiffsCompleted",12); + } boolean historyGrew = snap.get("messages").getAsInt()>initialMessages; record("P_realHistoryGrew",historyGrew); record("P_realTypingSurvived",typingHealthy); record("P_final",snap); boolean firstCompleted=completed; + boolean firstStreamed=snap.get("partial").getAsInt()>initialPartial; + record("P_firstTurnStreamedWhileTyping",firstStreamed); + boolean followupStreamed=false; if (completed) { mark("P-followup-user-turn"); - Toolkit.getDefaultToolkit().getSystemClipboard().setContents(new java.awt.datatransfer.StringSelection( - "Run printf SOLHEIM_THIRD; sleep 5 in this workspace and finish with attempt_completion. Do not change files."),null); + String followupPrompt = fixtureMode ? "Stream a second fixture response and finish with attempt_completion." : windowsLive ? "Run execute_command with exactly: " + windowsCommand("third") + ". Do not change any files. Finish with attempt_completion." : "Run printf SOLHEIM_THIRD; sleep 5 in this workspace and finish with attempt_completion. Do not change files."; + // A new user turn starts with a deliberate click. Never refocus inside + // either draft-preservation loop or repair a failed typing sample. + clickInBrowser(browser, snap.getAsJsonObject("rect")); + Toolkit.getDefaultToolkit().getSystemClipboard().setContents(new java.awt.datatransfer.StringSelection(followupPrompt),null); robot.keyPress(KeyEvent.VK_CONTROL);robot.keyPress(KeyEvent.VK_A);robot.keyRelease(KeyEvent.VK_A);robot.keyRelease(KeyEvent.VK_CONTROL); robot.keyPress(KeyEvent.VK_CONTROL);robot.keyPress(KeyEvent.VK_V);robot.keyRelease(KeyEvent.VK_V);robot.keyRelease(KeyEvent.VK_CONTROL); Thread.sleep(800); + snap=realSnapshot(webview,title); + record("P_followupPromptEntered",snap); + if (!followupPrompt.equals(snap.get("value").isJsonNull()?null:snap.get("value").getAsString())) { + fail("followupPromptMismatch", "Native paste did not enter the requested followup; refusing to submit a draft"); + return; + } robot.keyPress(KeyEvent.VK_ENTER);robot.keyRelease(KeyEvent.VK_ENTER); Thread.sleep(2000); expected="";completed=false; + initialPartial=realSnapshot(webview,title).get("partial").getAsInt(); deadline=System.currentTimeMillis()+150000; int followup=0; while(System.currentTimeMillis()2) {completed=true;break;} if (!answerCommandAsk(webview, snap, ask)) return; } + followupStreamed=snap.get("partial").getAsInt()>initialPartial; } + boolean commandsCovered = fixtureMode || (windowsLive + ? approvedCommands.containsAll(java.util.Arrays.asList(windowsCommand("first"), windowsCommand("second"), windowsCommand("third"))) + : java.util.Arrays.asList("SOLHEIM_FIRST", "SOLHEIM_SECOND", "SOLHEIM_THIRD").stream() + .allMatch(marker -> approvedCommands.stream().anyMatch(command -> command.contains(marker)))); + record("P_requiredCommandsApproved",commandsCovered); record("P_followupCompleted",completed); record("P_bothTurnsTypingSurvived",typingHealthy); - boolean fixtureChanged = Files.readString(Paths.get(project.getBasePath(),"src/Main.java")).contains("solheim_e2e"); - record("P_fixtureChanged",fixtureChanged); + record("P_bothTurnsFocusPreserved",focusHealthy); + record("P_followupStreamedWhileTyping",followupStreamed); + boolean fixtureChanged = fixtureMode || Files.readString(Paths.get(project.getBasePath(),"src/Main.java")).contains("solheim_e2e"); + if (!fixtureMode) record("P_fixtureChanged",fixtureChanged); + record("providerMode",fixtureMode ? "local-streaming-fixture" : "live-solheim"); screenshot("p-real-conversation"); - finish("testP-real-solheim-conversation", firstCompleted&&completed&&typingHealthy&&historyGrew&&fixtureChanged?"PASS":"FAIL-REAL-CONVERSATION"); + finish(fixtureMode ? "testP-streaming-fixture" : "testP-real-solheim-conversation", firstCompleted&&completed&&commandsCovered&&typingHealthy&&focusHealthy&&firstStreamed&&followupStreamed&&historyGrew&&fixtureChanged?"PASS":"FAIL-REAL-CONVERSATION"); } private void clickInBrowser(Object jbCefBrowser, JsonObject rect) throws Exception { @@ -369,7 +518,7 @@ private void clickInBrowser(Object jbCefBrowser, JsonObject rect) throws Excepti if (loc[0] == null) { throw new IllegalStateException("browser component not showing"); } - int cx = loc[0].x + rect.get("x").getAsInt() + rect.get("w").getAsInt() / 2; + int cx = loc[0].x + rect.get("x").getAsInt() + Math.min(20, rect.get("w").getAsInt() / 2); int cy = loc[0].y + rect.get("y").getAsInt() + rect.get("h").getAsInt() / 2; Robot robot = new Robot(); robot.setAutoDelay(60); @@ -393,6 +542,68 @@ private void typeText(String text) throws Exception { log("typed text of length " + text.length()); } + // Exercise showTextDocument followed by diff, as used by live apply_diff. + // Both production calls run against fresh files while native keys go + // to the real chat. Reflection keeps Kotlin/IDE classes in their own loaders. + private void openPreservingEditors(Project project, int index) throws Exception { + Path dir = Paths.get(project.getBasePath(), ".focus-diffs"); + Path left = dir.resolve("before-" + index + ".java"); + Path right = dir.resolve("after-" + index + ".java"); + Class commandClass = zooClass("org.zoocode.jetbrains.editor.OpenEditorAPICommands"); + Object command = commandClass.getConstructor(Project.class).newInstance(project); + Method method = java.util.Arrays.stream(commandClass.getMethods()) + .filter(m -> m.getName().equals("workbench_diff")).findFirst().orElseThrow(); + java.util.Map leftUri = java.util.Map.of("scheme","file","path",left.toString().replace('\\','/')); + java.util.Map rightUri = java.util.Map.of("scheme","file","path",right.toString().replace('\\','/')); + Class textEditorsClass = zooClass("org.zoocode.jetbrains.actors.MainThreadTextEditors"); + Object textEditors = textEditorsClass.getConstructor(Project.class).newInstance(project); + Method show = java.util.Arrays.stream(textEditorsClass.getMethods()) + .filter(m -> m.getName().equals("tryShowTextDocument")).findFirst().orElseThrow(); + invokeSuspend(textEditors, show, rightUri, java.util.Map.of("preserveFocus",true)); + runOnEdt(() -> { + boolean opened = java.util.Arrays.stream(com.intellij.openapi.fileEditor.FileEditorManager.getInstance(project).getOpenFiles()) + .anyMatch(file -> file.isInLocalFileSystem() && file.toNioPath().equals(right)); + if (!opened) throw new IllegalStateException("Production call did not open file " + index); + }); + log("preserving file completed: " + index); + invokeSuspend(command, method, leftUri, rightUri, "Input preservation " + index, + java.util.Map.of("preserveFocus",true)); + runOnEdt(() -> { + boolean opened = false; + for (com.intellij.openapi.vfs.VirtualFile file : com.intellij.openapi.fileEditor.FileEditorManager.getInstance(project).getOpenFiles()) { + if (file.getClass().getName().equals("com.intellij.diff.editor.ChainDiffVirtualFile")) { + Object chain = invoke(file, "getChain"); + for (Object request : (List) invoke(chain, "getRequests")) { + if (("Input preservation " + index).equals(invoke(request, "getName"))) opened = true; + } + } + } + if (!opened) throw new IllegalStateException("Production command did not open diff " + index); + }); + log("preserving diff completed: " + index); + } + + private Object invokeSuspend(Object target, Method method, Object... arguments) throws Exception { + Class continuationClass = method.getParameterTypes()[arguments.length]; + ClassLoader loader = continuationClass.getClassLoader(); + Object context = Class.forName("kotlin.coroutines.EmptyCoroutineContext", true, loader).getField("INSTANCE").get(null); + java.util.concurrent.CompletableFuture done = new java.util.concurrent.CompletableFuture<>(); + Object continuation = Proxy.newProxyInstance(loader, new Class[]{continuationClass}, (proxy, m, args) -> { + if (m.getName().equals("getContext")) return context; + if (m.getName().equals("resumeWith")) { done.complete(args[0]); return null; } + if (m.getName().equals("toString")) return "FocusEditorContinuation"; + if (m.getName().equals("hashCode")) return System.identityHashCode(proxy); + if (m.getName().equals("equals")) return proxy == args[0]; + return null; + }); + Object[] parameters = java.util.Arrays.copyOf(arguments, arguments.length + 1); + parameters[arguments.length] = continuation; + Object result = method.invoke(target, parameters); + if (result != null && result.toString().equals("COROUTINE_SUSPENDED")) result = done.get(60, TimeUnit.SECONDS); + Class.forName("kotlin.ResultKt", true, loader).getMethod("throwOnFailure", Object.class).invoke(null, result); + return result; + } + private String swingFocusOwnerClass() { final String[] r = {null}; runOnEdt(() -> { @@ -416,8 +627,9 @@ private String swingFocusOwnerIdentity() { private void dismissStartupDialogs(int seconds) { long deadline = System.currentTimeMillis() + seconds * 1000L; - while (System.currentTimeMillis() < deadline) { + while (!startupComplete && System.currentTimeMillis() < deadline) { runOnEdt(() -> { + if (startupComplete) return; for (Window w : Window.getWindows()) { if (!(w instanceof Dialog) || !w.isShowing() || (w instanceof JFrame)) { continue; @@ -434,6 +646,13 @@ private void dismissStartupDialogs(int seconds) { } private void handleDialog(Dialog d) { + com.intellij.openapi.ui.DialogWrapper wrapper = com.intellij.openapi.ui.DialogWrapper.findInstance(d); + if (wrapper != null && (wrapper.getClass().getName().startsWith("com.intellij.llmInstaller.ui.promo") || + containsAiPromotion(d))) { + log("Cancelling startup AI promotion: " + wrapper.getClass().getName()); + wrapper.close(com.intellij.openapi.ui.DialogWrapper.CANCEL_EXIT_CODE); + return; + } List buttons = new ArrayList<>(); List boxes = new ArrayList<>(); collect(d, buttons, boxes); @@ -444,7 +663,7 @@ private void handleDialog(Dialog d) { log("dialog checkbox selected: " + box.getText()); } } - String[] prefer = {"accept", "agree", "continue", "don't send", "dont send", "start", "skip", "no thanks", "got it", "close", "ok"}; + String[] prefer = {"don't send", "dont send", "skip", "no thanks", "got it", "close", "ok", "accept", "agree", "continue"}; for (String p : prefer) { for (JButton b : buttons) { String t = (b.getText() == null ? "" : b.getText()).toLowerCase(); @@ -457,6 +676,21 @@ private void handleDialog(Dialog d) { } } + private boolean containsAiPromotion(Component component) { + if (component.getClass().getName().startsWith("com.intellij.llmInstaller.ui.promo")) return true; + if (component instanceof JLabel) { + String text = ((JLabel) component).getText(); + if (text != null && (text.contains("WebStorm Goes AI") || + text.contains("Unlock next-level development with free AI"))) return true; + } + if (component instanceof Container) { + for (Component child : ((Container) component).getComponents()) { + if (containsAiPromotion(child)) return true; + } + } + return false; + } + private void collect(Component c, List buttons, List boxes) { if (c instanceof JButton) { buttons.add((JButton) c); @@ -603,17 +837,27 @@ private void record(String key, Object value) { */ private boolean answerCommandAsk(Object webview, JsonObject snap, String ask) throws Exception { if (snap.get("lastPartial").getAsBoolean()) return true; + if ("mistake_limit_reached".equals(ask)) { + fail("modelMistakeLimit", "Provider conversation stopped after repeated tool mistakes; required coverage is incomplete. " + + snap.get("lastText").getAsString()); + return false; + } if ("command".equals(ask)) { JsonElement text = snap.get("lastText"); JsonElement tsValue = snap.get("lastTs"); String command = text == null || text.isJsonNull() ? "" : text.getAsString().trim(); - if (!FIXTURE_COMMAND.matcher(command).matches()) { + boolean allowed = windowsLive + ? java.util.Arrays.asList(windowsCommand("first"), windowsCommand("second"), windowsCommand("third")).contains(command) + : FIXTURE_COMMAND.matcher(command).matches(); + if (!allowed) { fail("unexpectedCommand", "Refused a command outside the fixture: " + command); return false; } long ts = tsValue == null || tsValue.isJsonNull() ? -1 : tsValue.getAsLong(); if (ts >= 0 && ts == lastApprovedTs) return true; lastApprovedTs = ts; + approvedCommands.add(command); + record("P_approvedCommand", command); } else if (!"command_output".equals(ask)) { return true; } @@ -624,6 +868,7 @@ private boolean answerCommandAsk(Object webview, JsonObject snap, String ask) th private void fail(String reason, String detail) { log("FAIL reason=" + reason + " detail=" + detail); record("failReason", reason + ": " + detail); + screenshot("failure"); finish("test", "FAIL"); } diff --git a/jetbrains_plugin/scripts/ui-harness/README.md b/jetbrains_plugin/scripts/ui-harness/README.md index 88b9779..89c045a 100644 --- a/jetbrains_plugin/scripts/ui-harness/README.md +++ b/jetbrains_plugin/scripts/ui-harness/README.md @@ -41,7 +41,13 @@ The driver configures `qwen3.8-27b`, submits prompts through the real React textarea using Robot/clipboard, and types drafts during two model turns. The model runs commands, reads a file, and applies a diff in the throwaway workspace. Snapshot replies are correlated by request ID. PASS requires both -turns completed, exact draft preservation, history growth, and a changed fixture. +turns completed, exact draft preservation, textarea and browser-window focus at +every typing sample, streamed partial updates during each typing loop, history +growth, and a changed fixture. Focus is established once before the first typing +loop; the driver never refocuses to repair a failing sample. These assertions +cover [issue #5](https://github.com/Zoo-Code-Org/Zoo-Code-Jetbrains/issues/5). +A sampled focus check cannot detect a blur and recovery between samples; exact +draft comparisons also catch dropped keys when sampled focus stays intact. No fake input, fake history, or scripted model responses are used. The profile auto-approves workspace reads and writes. The driver answers a command @@ -51,8 +57,10 @@ request only when the command is one of the three fixture commands, for example separate from a real project. Stop the relay, unset the token, and remove the private key file afterward. The disposable profile contains the now-expired local token; remove it when -finished with the evidence. Check traces as well as status: a pass does not -clear the intermittent first-editor key-delivery gap recorded in +finished with the evidence. Check traces as well as status: a single pass is +insufficient for an intermittent input problem. Hosted Windows Server results +support the fix, but the linked run predates the latest workflow, harness, Kotlin, +and relay changes. Windows 11 remains unverified. See [the validation report](../../../docs/validation/chat-focus-preservation.md). Relay access tests, without provider calls: @@ -92,3 +100,168 @@ cause, and does not clear the intermittent draft loss described in Results vary with window manager, keyboard layout, and timing. Compare control and churn modes on the same host in the same session; a single run decides nothing. + +## Windows GitHub Actions regression test (issue #5) + +`.github/workflows/windows-chat-focus.yml` builds the plugin from the tested +checkout with Zoo Code extension **3.84.0**, then runs the same Java driver on +`windows-2022` with IntelliJ **2025.3** and **2026.2.1**. It runs for pull requests +that change the harness or relevant implementation, and supports manual dispatch +once the workflow exists on the default branch. It does not publish a plugin. +The IDE archives include JBR/JCEF; their SHA-256 is checked against JetBrains' +published checksum, pinned in the workflow matrix. The run identity records the IDE build, plugin digest, OS, +runner image, and tested commit. + +The scenario uses the real bundled extension, React input and history, JCEF, +and native `java.awt.Robot` key events. Only the OpenAI-compatible provider is +simulated by `streaming_fixture.py`, bound to loopback. No API credentials are +needed. Each response emits 100 text deltas over about 25 seconds followed by a +native `attempt_completion` tool call. Two turns must complete, streaming events +and history must grow, and every draft character and sampled focus check must +survive. Windows fixture runs also open 12 fresh Java files, each followed by a diff, +through the production APIs while typing; every file and diff tab must open. Test files are prepared +before typing so filesystem discovery is separate from editor-loading coverage. +The driver never repairs focus during the typing loops. + +Before chat testing, a standalone Swing window must receive exact native typing; +the real React textarea must then receive a separate native typing smoke test. +A missing interactive desktop or failed smoke test fails the job. It is not +reported as a pass or a skipped focus regression. Startup browser selection waits +for a loaded, mounted instance after indexing, since startup can retire an earlier +webview. + +The `windows-chat-focus--` artifact contains desktop and chat +screenshots, `run-identity.json`, `evidence.json`, `progress.log`, IDE logs, +provider stream metadata, and `runner-error.txt` on runner failures. A missing +status, IDE crash, timeout, dropped key, or failed focus assertion fails the job. +Sampled focus checks can miss brief transitions between samples; exact text +comparisons still catch dropped keys. Correlated snapshots use JCEF console +and title callbacks; repeating an unanswered observation does not type or focus. +The fixture covers streaming and preserving file and diff opens. Use the live Solheim +battery for terminal creation and real model/tool behavior. + +For a local Windows desktop session, start PowerShell (not a Windows service): + +```powershell +./run-windows.ps1 -IdeDir C:/tools/idea -PluginZip C:/build/ZooCode.zip ` + -EvidenceDir C:/temp/focus-evidence-1 +``` + +Prerequisites are Python 3, a built **release** plugin zip, and an IntelliJ Windows +ZIP distribution with its bundled JBR. Each run needs a fresh evidence directory. +The runner starts/stops its own fixture and IDE and removes the disposable profile. +Keep the desktop unlocked and avoid interacting with it while tests run. Hosted +`windows-2022` is Windows Server x64, not the reporter's Windows 11/WebStorm setup; +a pass is evidence for the tested configuration only. The same runner can be used +on a self-hosted Windows 11 desktop with IntelliJ. + +Fixture HTTP integration tests: + +```sh +python3 -m unittest -v test_streaming_fixture.py +``` + +The fixture can also exercise the conversation on Linux using `run-solheim.sh` +with `HARNESS_FIXTURE=true` (the relay token is then unnecessary). Start +`python3 streaming_fixture.py --log /tmp/focus-provider.jsonl` first, then use +the IDE/plugin/window-manager variables from the manual setup above. Set the +fixture flag through this environment variable, not `JAVA_TOOL_OPTIONS`, which +can display an IDE warning balloon covering the input. + +## Maintainer-approved live Solheim testing on Windows + +The `live-solheim` job in the Windows focus workflow uses the protected GitHub +Environment **solheim** with the Environment secret **SOLHEIM_API_KEY**. Required +reviewers must approve the exact tested commit before it receives the key. Review +the workflow and harness script changes before approving: PR-head scripts receive +the credential in this job. The environment currently allows self-approval and +administrator bypass; approval does not guarantee an independent review. It +runs only when a PR has the **live-solheim** label, or a manual dispatch selects +`live_solheim`. Manual dispatch is available after this workflow is on the +default branch; the label enables testing this PR before merging. + +The job first requires the credential-free matrix to pass. It downloads pinned +WebStorm **2026.2.1**, then runs three fresh two-turn conversations with the real +Solheim `qwen3.8-27b` provider. Each profile requires exact native draft typing and +sampled focus preservation during streaming. Fixed Windows PowerShell scripts +run through the real terminal, sleep while the driver types, read the fixture, +and write completion markers. The model must read `src/Main.java` and use +`apply_diff` to change `throwaway` to `solheim_e2e`. + +The driver waits for a non-partial completed-task ask before the next turn, +clicks the input once to begin that deliberate user action, and verifies that +native paste entered the exact follow-up prompt before pressing Enter. It fails +instead of submitting a preserved draft as a prompt. All three expected commands +must also have been approved; no focus is repaired inside the typing loops. + +PASS additionally requires all three terminal completion markers, provider tool +calls for `execute_command`, `read_file`, `apply_diff`, and `attempt_completion`, +a changed fixture, and IDE log evidence of a completed diff with +`preserveFocus=true`. Failures stop the sequence; they are not retried or dropped. +`live-validation.json` records the extra coverage checks. A live run can fail +because the model did not perform the required tools, even if focus was healthy; +check the artifacts before classifying a failure as the product bug. + +The runner creates a random relay token and a temporary key file in a directory +restricted to the current Windows user. It removes the upstream key from the +child-process environment before launching any processes. The key file is +removed after the relay becomes healthy, and the IDE receives only the local +relay token. The relay retains the upstream key in memory and logs metadata, +not request/response content. Only `SOLHEIM_API_KEY` is a persistent GitHub secret; +`SOLHEIM_KEY_FILE`, `SOLHEIM_PROXY_LOG`, and `SOLHEIM_RELAY_TOKEN` are generated +per run. Do not add them as GitHub secrets. + +Hosted runs use Windows Server x64. For the original OS, register an interactive +Windows 11 x64 self-hosted runner with labels `Windows`, `X64`, and +`windows11-x64`, then select `live_runner=windows11-x64` on manual dispatch. +The runner checks Windows client build >= 22000 and x64 architecture before +claiming this configuration. Keep the desktop unlocked and run the agent as a +foreground process rather than a service. The private profile and fixture do +not isolate the OS: use a dedicated test machine. The same test is available +locally: + +```powershell +# Set SOLHEIM_API_KEY in this process without putting its value in shell history. +./run-windows.ps1 -IdeDir C:/tools/WebStorm-2026.2.1 ` + -PluginZip C:/build/ZooCode.zip -Provider solheim -RequireWindows11 ` + -DeadlineSeconds 780 -EvidenceDir C:/temp/live-focus-1 +``` + +The linked hosted Windows Server run supports the fix. Close #5 only after +the reporter or a Windows 11 run confirms it; a Windows Server run alone does not +verify Windows 11. Later workflow, harness, Kotlin, and relay changes require +fresh CI validation. + +## Local WebStorm validation + +The Linux runner selects the native launcher and configuration variables from +`product-info.json`; both IntelliJ and WebStorm are supported. Chromium uses +temporary-file backing to accommodate small container shared-memory mounts. To reuse a +legitimately activated local WebStorm license or evaluation in a disposable +profile, set `HARNESS_LICENSE_FILE` to its existing `webstorm.key`. The runner +copies it with mode 0600 into the private profile, never into evidence. This +requires normal license/trial activation; it does not create or alter a license. + +```sh +IDE_DIR=/path/to/WebStorm-262.9437.145 \ +PLUGIN_ZIP=/path/to/ZooCode.zip \ +HARNESS_LICENSE_FILE=/path/to/private/webstorm.key \ +HARNESS_WM_BIN=/path/to/openbox \ +HARNESS_FIXTURE=true bash run-solheim.sh +``` + +Start `streaming_fixture.py` first for the command above. Add +`HARNESS_DIFF_STRESS=true` to exercise the same 12-file/12-diff regression as Windows CI. +For live testing, use the private relay setup and omit `HARNESS_FIXTURE=true`. + +Startup now waits for modal/licensing windows to close. The generic startup +handler does not choose “Start” actions such as a quick tour. The AI promotion +is cancelled by its DialogWrapper type. Before typing anything, the driver +clicks an unobscured point near the left edge of the input and checks that native +focus arrived; setup clicks are bounded and recorded as `P_initialFocusClicks`. +There are no setup clicks or dialog handlers inside either draft-preservation +loop. Exact draft and focus assertions are unchanged. + +The external Linux agreement helper only targets named IDE agreement windows +and stops when the in-IDE driver starts. It no longer clicks arbitrary windows +by size during a conversation. diff --git a/jetbrains_plugin/scripts/ui-harness/run-solheim.sh b/jetbrains_plugin/scripts/ui-harness/run-solheim.sh index 3997fc9..537f52c 100644 --- a/jetbrains_plugin/scripts/ui-harness/run-solheim.sh +++ b/jetbrains_plugin/scripts/ui-harness/run-solheim.sh @@ -2,10 +2,18 @@ set -uo pipefail SOURCE_DIR="$(cd "$(dirname "$0")" && pwd)" : "${IDE_DIR:?Set IDE_DIR to a JCEF-enabled IntelliJ installation}" : "${PLUGIN_ZIP:?Set PLUGIN_ZIP to the built ZooCode distribution}" -: "${SOLHEIM_RELAY_TOKEN:?Set the same random token as the relay}" +if [ "${HARNESS_FIXTURE:-false}" != true ]; then + : "${SOLHEIM_RELAY_TOKEN:?Set the same random token as the relay}" +fi : "${HARNESS_WM_BIN:?Set HARNESS_WM_BIN to openbox or an equivalent wrapper}" IDE_DIR="$(realpath "$IDE_DIR")" PLUGIN_ZIP="$(realpath "$PLUGIN_ZIP")" +PRODUCT_CODE="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["productCode"])' "$IDE_DIR/product-info.json")" +case "$PRODUCT_CODE" in + WS) IDE_LAUNCHER=webstorm ;; + IU|IC) IDE_LAUNCHER=idea ;; + *) echo "Unsupported harness IDE product: $PRODUCT_CODE" >&2; exit 1 ;; +esac TEST=p H="$(mktemp -d "${TMPDIR:-/tmp}/zoo-solheim-e2e.XXXXXXXX")" echo "Disposable harness and evidence: $H" @@ -20,6 +28,13 @@ DEADLINE="${DEADLINE_SECONDS:-780}" rm -rf "$CFG" "$EV" mkdir -p "$CFG/config" "$CFG/system" "$CFG/log" "$CFG/plugins" "$EV" "$H/props" +if [ -n "${HARNESS_LICENSE_FILE:-}" ]; then + [ "$PRODUCT_CODE" = WS ] || { echo 'HARNESS_LICENSE_FILE currently supports WebStorm only' >&2; exit 1; } + # Reuse a legitimately activated local license/trial in a fresh test profile. + # The profile is private; license contents are never copied into evidence. + chmod 700 "$CFG/config" + install -m 600 -- "$HARNESS_LICENSE_FILE" "$CFG/config/webstorm.key" || exit 1 +fi XVFB_NUM=$((300 + RANDOM % 500)) Xvfb ":$XVFB_NUM" -screen 0 1600x1000x24 -nolisten tcp > "$EV/xvfb.log" 2>&1 & @@ -63,8 +78,19 @@ EOF echo "-XX:ActiveProcessorCount=$HARNESS_CPU_COUNT" fi echo "-Didea.trust.all.projects=true" + echo "-Didea.initially.ask.config=false" + echo "-Djb.consents.confirmation.enabled=false" + echo '-Djb.privacy.policy.text=' echo "-Dide.show.tips.on.startup.default.value=false" + # Containers often share a 64 MiB /dev/shm with other browsers. Let Chromium + # back its buffers with temporary files instead of exhausting that mount. + echo "-Dide.browser.jcef.extra.args=--disable-dev-shm-usage" echo "-Dzoo.harness.mode=${TEST^^}" + echo "-Dzoo.harness.fixture=${HARNESS_FIXTURE:-false}" + echo "-Dzoo.harness.diffStress=${HARNESS_DIFF_STRESS:-false}" + if [ "${HARNESS_DIFF_STRESS:-false}" = true ]; then + echo "-Didea.log.debug.categories=#com.intellij.openapi.wm.impl.FocusManagerImpl,jb.focus.requests" + fi echo "-Dzoo.harness.evidence=$EV" } > "$H/props/vm-$TEST.vmoptions" @@ -79,15 +105,21 @@ fi export IDEA_PROPERTIES="$H/props/idea-$TEST.properties" export IDEA_VM_OPTIONS="$H/props/vm-$TEST.vmoptions" +if [ "$PRODUCT_CODE" = WS ]; then + export WEBIDE_PROPERTIES="$H/props/idea-$TEST.properties" + export WEBIDE_VM_OPTIONS="$H/props/vm-$TEST.vmoptions" +fi click_startup_dialogs() { # The user-agreement dialog is borderless, ~600x460, centered on the # 1600x1000 Xvfb screen. It blocks frame creation before any in-IDE listener # runs, so dismiss it externally: select the confirm checkbox, then Continue. for _ in $(seq 1 30); do - for wid in $(xdotool search --onlyvisible . 2>/dev/null); do + [ -f "$EV/progress.log" ] && return + for wid in $(xdotool search --onlyvisible --name '(IntelliJ IDEA|WebStorm) User Agreement' 2>/dev/null); do geom=$(xdotool getwindowgeometry --shell "$wid" 2>/dev/null) || continue eval "$geom" + [ -f "$EV/progress.log" ] && return if [ "${WIDTH:-0}" -ge 550 ] && [ "${WIDTH:-0}" -le 700 ] && [ "${HEIGHT:-0}" -ge 400 ] && [ "${HEIGHT:-0}" -le 520 ]; then echo "harness: clicking startup agreement dialog wid=$wid ${WIDTH}x${HEIGHT}+${X}+${Y}" xdotool mousemove $((X + 40)) $((Y + HEIGHT * 85 / 100)) click 1 @@ -104,7 +136,7 @@ CLICKER_PID=$! ( cd "$IDE_DIR/bin" - exec ./idea "$H/workspace/throwaway-project" + exec "./$IDE_LAUNCHER" "$H/workspace/throwaway-project" ) > "$EV/launcher-stdout.log" 2>&1 & IDE_PID=$! diff --git a/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 b/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 new file mode 100644 index 0000000..f7c4cf8 --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 @@ -0,0 +1,266 @@ +# Run only in an interactive Windows desktop session. Live mode uses the Solheim relay. +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string]$IdeDir, + [Parameter(Mandatory = $true)][string]$PluginZip, + [Parameter(Mandatory = $true)][string]$EvidenceDir, + [ValidateRange(60, 1800)][int]$DeadlineSeconds = 600, + [ValidateSet('fixture', 'solheim')][string]$Provider = 'fixture', + [switch]$RequireWindows11 +) +$ErrorActionPreference = 'Stop' +$IdeDir = (Resolve-Path $IdeDir).Path +$PluginZip = (Resolve-Path $PluginZip).Path +$EvidenceDir = [IO.Path]::GetFullPath($EvidenceDir) +New-Item -ItemType Directory -Force $EvidenceDir | Out-Null +if (Test-Path (Join-Path $EvidenceDir 'status.txt')) { throw 'Use a fresh evidence directory for each run.' } +$runDir = Join-Path ([IO.Path]::GetTempPath()) ('zoo-focus-' + [guid]::NewGuid().ToString('N')) +New-Item -ItemType Directory $runDir | Out-Null +$ideProcess = $null +$fixtureProcess = $null +$live = $Provider -eq 'solheim' +$originalRelayToken = $env:SOLHEIM_RELAY_TOKEN +$originalKeyFile = $env:SOLHEIM_KEY_FILE +$originalProxyLog = $env:SOLHEIM_PROXY_LOG +if ($live) { + if ([string]::IsNullOrWhiteSpace($env:SOLHEIM_API_KEY)) { throw 'Solheim mode requires SOLHEIM_API_KEY' } + # Remove the upstream key from the environment before starting any children. + $upstreamKey = $env:SOLHEIM_API_KEY + $env:SOLHEIM_API_KEY = $null +} +$originalProperties = $env:IDEA_PROPERTIES +$originalOptions = $env:IDEA_VM_OPTIONS +$original64Options = $env:IDEA64_VM_OPTIONS +$originalWebideProperties = $env:WEBIDE_PROPERTIES +$originalWebideOptions = $env:WEBIDE_VM_OPTIONS + +function Invoke-CheckedNative { + param([string]$Executable, [string[]]$Arguments) + & $Executable @Arguments + if ($LASTEXITCODE -ne 0) { throw "$Executable failed with exit code $LASTEXITCODE" } +} + +try { + $windows = Get-CimInstance Win32_OperatingSystem + if ($RequireWindows11 -and ($windows.ProductType -ne 1 -or [int]$windows.BuildNumber -lt 22000 -or + [Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() -ne 'X64')) { + throw 'This reproduction requires Windows 11 x64, not Windows Server or ARM64' + } + $identity = [ordered]@{ + commit = (& git rev-parse HEAD) + os = [Environment]::OSVersion.VersionString + windowsCaption = $windows.Caption + windowsBuild = $windows.BuildNumber + windowsProductType = $windows.ProductType + architecture = [Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() + interactive = [Environment]::UserInteractive + sessionId = (Get-Process -Id $PID).SessionId + pluginSha256 = (Get-FileHash $PluginZip -Algorithm SHA256).Hash + ide = (Get-Content (Join-Path $IdeDir 'product-info.json') -Raw | ConvertFrom-Json) + provider = $Provider + runnerImage = $env:ImageVersion + run = "$env:GITHUB_SERVER_URL/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID" + } + $identity | ConvertTo-Json -Depth 10 | Set-Content (Join-Path $EvidenceDir 'run-identity.json') + if (-not [Environment]::UserInteractive -or (Get-Process -Id $PID).SessionId -eq 0) { + throw 'A logged-in interactive desktop is required; do not run the agent as a Windows service.' + } + # JetBrains recommends dismissing the Windows Start overlay for UI tests. + Stop-Process -Name StartMenuExperienceHost -Force -ErrorAction SilentlyContinue + $javaBin = Join-Path $IdeDir 'jbr/bin' + $classes = Join-Path $runDir 'driver-classes' + New-Item -ItemType Directory -Force (Join-Path $classes 'META-INF') | Out-Null + Invoke-CheckedNative "$javaBin/javac.exe" @('-cp', "$IdeDir/lib/*", '-d', $classes, + "$PSScriptRoot/HarnessDriver.java", "$PSScriptRoot/DesktopTypingProbe.java") + # Separate desktop setup failures from IDE/JCEF focus regressions. + $probe = Start-Process -FilePath "$javaBin/java.exe" -ArgumentList @( + '-cp', "`"$classes`"", 'DesktopTypingProbe', "`"$EvidenceDir`"" + ) -PassThru -RedirectStandardOutput "$EvidenceDir/desktop-stdout.log" -RedirectStandardError "$EvidenceDir/desktop-stderr.log" + if (-not $probe.WaitForExit(30000)) { + & taskkill.exe /PID $probe.Id /T /F | Out-Null + throw 'Native desktop typing preflight timed out' + } + if ($probe.ExitCode -ne 0) { throw "Native desktop typing preflight failed: $($probe.ExitCode)" } + Copy-Item "$PSScriptRoot/plugin.xml" (Join-Path $classes 'META-INF/plugin.xml') + $plugins = Join-Path $runDir 'plugins' + $driverLib = Join-Path $plugins 'harness-driver/lib' + New-Item -ItemType Directory -Force $driverLib | Out-Null + # The IDE's bundled JBR includes javac but may omit the jar executable. + # A plugin JAR is a ZIP with classes and META-INF/plugin.xml at its root. + [IO.Compression.ZipFile]::CreateFromDirectory($classes, "$driverLib/driver.jar") + Expand-Archive -LiteralPath $PluginZip -DestinationPath $plugins + if (-not (Test-Path "$plugins/ZooCode/runtime/extension.js")) { + throw 'Distribution must contain the release extension-host runtime; build with debugMode=release.' + } + $project = Join-Path $runDir 'workspace' + New-Item -ItemType Directory -Force "$project/src" | Out-Null + 'public class Main { public static void main(String[] args) { System.out.println("throwaway"); } }' | Set-Content "$project/src/Main.java" + if ($live) { + New-Item -ItemType Directory "$project/.focus-ci" | Out-Null + foreach ($step in @('first', 'second', 'third')) { + # Fixed scripts: the model only chooses when to invoke them. + @" +`$ErrorActionPreference = 'Stop' +Write-Output SOLHEIM_$($step.ToUpper()) +Start-Sleep -Seconds 8 +Get-Content src/Main.java +Set-Content .focus-ci/$step.done 'completed' +"@ | Set-Content "$project/.focus-ci/$step.ps1" + } + } + $properties = Join-Path $runDir 'idea.properties' + $profilePath = $runDir.Replace('\', '/') + @" +idea.config.path=$profilePath/config +idea.system.path=$profilePath/system +idea.log.path=$profilePath/log +idea.plugins.path=$profilePath/plugins +"@ | Set-Content $properties + $options = Join-Path $runDir 'idea.vmoptions' + $evidencePath = $EvidenceDir.Replace('\', '/') + @" +-Xmx1536m +-XX:ActiveProcessorCount=4 +-Didea.trust.all.projects=true +-Didea.initially.ask.config=false +-Djb.consents.confirmation.enabled=false +-Djb.privacy.policy.text= +-Dide.show.tips.on.startup.default.value=false +-Dzoo.harness.mode=P +-Dzoo.harness.fixture=$((-not $live).ToString().ToLowerInvariant()) +-Dzoo.harness.diffStress=$((-not $live).ToString().ToLowerInvariant()) +-Dzoo.harness.evidence=$evidencePath +"@ | Set-Content $options + $env:IDEA_PROPERTIES = $properties + $env:IDEA_VM_OPTIONS = $options + $env:IDEA64_VM_OPTIONS = $options + + # Refuse to use a stale provider left by another run. + $listener = [Net.Sockets.TcpListener]::new([Net.IPAddress]::Loopback, 19487) + try { $listener.Start() } finally { $listener.Stop() } + # A separate visible Python console can cover the IDE and consume Robot keys. + # Keep the provider alive without creating a foreground test obstacle. + $providerScript = "$PSScriptRoot/streaming_fixture.py" + $providerArguments = @("`"$providerScript`"", '--log', "`"$EvidenceDir/provider.jsonl`"") + $readyUri = 'http://127.0.0.1:19487/health' + $readyHeaders = @{} + if ($live) { + $privateDir = New-Item -ItemType Directory "$runDir/private" + $acl = Get-Acl $privateDir.FullName + $acl.SetAccessRuleProtection($true, $false) + $rule = [Security.AccessControl.FileSystemAccessRule]::new( + [Security.Principal.WindowsIdentity]::GetCurrent().User, 'FullControl', + 'ContainerInherit,ObjectInherit', 'None', 'Allow') + $acl.AddAccessRule($rule) + Set-Acl -Path $privateDir.FullName -AclObject $acl + $env:SOLHEIM_KEY_FILE = "$runDir/private/key" + [IO.File]::WriteAllText($env:SOLHEIM_KEY_FILE, $upstreamKey) + $upstreamKey = $null + $env:SOLHEIM_RELAY_TOKEN = [Convert]::ToHexString([Security.Cryptography.RandomNumberGenerator]::GetBytes(32)) + $env:SOLHEIM_PROXY_LOG = "$EvidenceDir/provider.jsonl" + $providerArguments = @("`"$PSScriptRoot/solheim_proxy.py`"") + $readyUri = 'http://127.0.0.1:19487/v1/health' + $readyHeaders = @{ Authorization = "Bearer $env:SOLHEIM_RELAY_TOKEN" } + } + $fixtureProcess = Start-Process -FilePath 'python' -WindowStyle Hidden -ArgumentList $providerArguments -PassThru -RedirectStandardOutput "$EvidenceDir/provider-stdout.log" -RedirectStandardError "$EvidenceDir/provider-stderr.log" + $ready = $false + for ($attempt = 0; $attempt -lt 30; $attempt++) { + if ($fixtureProcess.HasExited) { throw 'Streaming fixture exited during startup' } + try { + $response = Invoke-WebRequest $readyUri -Headers $readyHeaders -UseBasicParsing -TimeoutSec 1 + if ($response.StatusCode -eq 200) { $ready = $true; break } + } catch { Start-Sleep -Milliseconds 250 } + } + if (-not $ready) { throw 'Local provider did not become healthy' } + if ($live) { + Remove-Item $env:SOLHEIM_KEY_FILE -Force + $env:SOLHEIM_KEY_FILE = $null + $env:SOLHEIM_PROXY_LOG = $null + } + $product = Get-Content "$IdeDir/product-info.json" -Raw | ConvertFrom-Json + $launcher = if ($product.productCode -eq 'WS') { 'webstorm64.exe' } else { 'idea64.exe' } + if ($product.productCode -eq 'WS') { + # WebStorm's launcher ignores IDEA_PROPERTIES and IDEA_VM_OPTIONS. + # Without these variables it opens a normal first-run profile, without + # our plugins, custom paths, or harness JVM properties. + $env:WEBIDE_PROPERTIES = $properties + $env:WEBIDE_VM_OPTIONS = $options + } + $ideProcess = Start-Process -FilePath "$IdeDir/bin/$launcher" -ArgumentList "`"$project`"" -PassThru ` + -RedirectStandardOutput "$EvidenceDir/launcher-stdout.log" -RedirectStandardError "$EvidenceDir/launcher-stderr.log" + $deadline = [DateTime]::UtcNow.AddSeconds($DeadlineSeconds) + $startupDeadline = [DateTime]::UtcNow.AddSeconds(120) + $statusPath = Join-Path $EvidenceDir 'status.txt' + while ([DateTime]::UtcNow -lt $deadline -and -not (Test-Path $statusPath)) { + if ($ideProcess.HasExited) { throw "IDE exited before producing evidence: $($ideProcess.ExitCode)" } + if ([DateTime]::UtcNow -gt $startupDeadline -and -not (Test-Path "$EvidenceDir/progress.log")) { + throw 'IDE harness did not start within 120 seconds; inspect launcher configuration and startup dialogs in runner-final.png' + } + Start-Sleep -Seconds 1 + } + if (-not (Test-Path $statusPath)) { throw "E2E timed out after $DeadlineSeconds seconds" } + $status = (Get-Content $statusPath -Raw).Trim() + if ($status -ne 'PASS') { + $failureReason = 'See evidence.json and progress.log' + $evidenceJson = Join-Path $EvidenceDir 'evidence.json' + if (Test-Path $evidenceJson) { + $checks = (Get-Content $evidenceJson -Raw | ConvertFrom-Json).checks + if ($checks.failReason) { $failureReason = $checks.failReason } + } + throw "Chat focus E2E result: ${status}: $failureReason" + } + $events = @(Get-Content "$EvidenceDir/provider.jsonl" | ForEach-Object { $_ | ConvertFrom-Json }) + if ($live) { + $responses = @($events | Where-Object event -eq 'response') + $tools = @($responses | ForEach-Object { $_.tools } | Sort-Object -Unique) + $requiredTools = @('execute_command', 'read_file', 'apply_diff', 'attempt_completion') + $missingTools = @($requiredTools | Where-Object { $_ -notin $tools }) + $missingCommands = @(@('first', 'second', 'third') | Where-Object { -not (Test-Path "$project/.focus-ci/$_.done") }) + $ideLog = Get-Content "$runDir/log/idea.log" -Raw + $validation = [ordered]@{ + missingTools = $missingTools + missingCommands = $missingCommands + preservingDiff = $ideLog.Contains('Opening diff, preserveFocus=true') -and $ideLog.Contains('Opening diff completed') + successfulResponses = @($responses | Where-Object status -eq 200).Count + failedResponses = @($responses | Where-Object status -ne 200).Count + } + $validation | ConvertTo-Json -Depth 5 | Set-Content "$EvidenceDir/live-validation.json" + if ($missingTools.Count -or $missingCommands.Count -or -not $validation.preservingDiff -or + $validation.successfulResponses -lt 2 -or $validation.failedResponses) { + throw 'Live terminal/diff/provider coverage incomplete; see live-validation.json' + } + Copy-Item "$project/src/Main.java" "$EvidenceDir/changed-fixture.java" + } elseif (@($events | Where-Object event -eq 'completed').Count -lt 2) { + throw 'Expected at least two completed provider streams' + } + Write-Output "PASS: provider=$Provider, native desktop/chat typing, two streamed turns, and required tool coverage" +} catch { + $_ | Out-String | Set-Content "$EvidenceDir/runner-error.txt" + throw +} finally { + try { + Add-Type -AssemblyName System.Windows.Forms + Add-Type -AssemblyName System.Drawing + $bounds = [Windows.Forms.SystemInformation]::VirtualScreen + $bitmap = [Drawing.Bitmap]::new($bounds.Width, $bounds.Height) + $graphics = [Drawing.Graphics]::FromImage($bitmap) + try { + $graphics.CopyFromScreen($bounds.Location, [Drawing.Point]::Empty, $bounds.Size) + $bitmap.Save("$EvidenceDir/runner-final.png", [Drawing.Imaging.ImageFormat]::Png) + } finally { $graphics.Dispose(); $bitmap.Dispose() } + } catch { Write-Warning "Could not capture final desktop: $_" } + if ($ideProcess -and -not $ideProcess.HasExited) { & taskkill.exe /PID $ideProcess.Id /T /F | Out-Null } + if ($fixtureProcess -and -not $fixtureProcess.HasExited) { Stop-Process -Id $fixtureProcess.Id -Force -ErrorAction SilentlyContinue } + if (Test-Path "$runDir/log") { Copy-Item "$runDir/log" "$EvidenceDir/ide-log" -Recurse -Force } + $upstreamKey = $null + $env:SOLHEIM_RELAY_TOKEN = $originalRelayToken + $env:SOLHEIM_KEY_FILE = $originalKeyFile + $env:SOLHEIM_PROXY_LOG = $originalProxyLog + $env:IDEA_PROPERTIES = $originalProperties + $env:IDEA_VM_OPTIONS = $originalOptions + $env:IDEA64_VM_OPTIONS = $original64Options + $env:WEBIDE_PROPERTIES = $originalWebideProperties + $env:WEBIDE_VM_OPTIONS = $originalWebideOptions + # Keep evidence; remove the disposable IDE profile and workspace. + Remove-Item $runDir -Recurse -Force -ErrorAction SilentlyContinue +} diff --git a/jetbrains_plugin/scripts/ui-harness/solheim_proxy.py b/jetbrains_plugin/scripts/ui-harness/solheim_proxy.py index fed0bf6..669102d 100644 --- a/jetbrains_plugin/scripts/ui-harness/solheim_proxy.py +++ b/jetbrains_plugin/scripts/ui-harness/solheim_proxy.py @@ -26,6 +26,15 @@ def record(**fields): output.write(json.dumps({'time': time.time(), **fields}) + '\n') +class RefuseRedirects(urllib.request.HTTPRedirectHandler): + def redirect_request(self, request, response, code, message, headers, new_url): + # Never send provider credentials to a redirected URL. + return None + + +UPSTREAM = urllib.request.build_opener(RefuseRedirects()) + + class Relay(http.server.BaseHTTPRequestHandler): protocol_version = 'HTTP/1.1' @@ -50,6 +59,13 @@ def relay(self): if not self.path.startswith('/v1/'): self.send_error(404) return + if self.command == 'GET' and self.path == '/v1/health': + # Local readiness only; never probe the paid upstream in a startup loop. + self.send_response(200) + self.send_header('Content-Length', '5') + self.end_headers() + self.wfile.write(b'ready') + return lengths = self.headers.get_all('Content-Length', []) length_text = lengths[0] if lengths else '0' if (len(lengths) > 1 or self.headers.get('Transfer-Encoding') is not None @@ -78,7 +94,7 @@ def relay(self): headers={'Authorization': 'Bearer ' + KEY, 'Content-Type': 'application/json'}) try: - response = urllib.request.urlopen(request, timeout=180) + response = UPSTREAM.open(request, timeout=180) except urllib.error.HTTPError as error: response = error except Exception as error: diff --git a/jetbrains_plugin/scripts/ui-harness/streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/streaming_fixture.py new file mode 100644 index 0000000..3dde3b1 --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/streaming_fixture.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Deterministic loopback OpenAI fixture for headful chat-focus CI tests. + +Only the provider is simulated. The extension, React history, JCEF, and native +keyboard delivery remain real. No commands or file edits are requested. +""" +import argparse +import http.server +import json +import threading +import time +from pathlib import Path + + +class Fixture(http.server.BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_GET(self): + if self.path == '/health': + self.send_response(200) + self.end_headers() + self.wfile.write(b'ready') + elif self.path == '/v1/models': + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps({'object': 'list', 'data': [ + {'id': 'focus-fixture', 'object': 'model', 'owned_by': 'fixture'} + ]}).encode()) + else: + self.send_error(404) + + def do_POST(self): + if self.path != '/v1/chat/completions': + self.send_error(404) + return + if self.headers.get('Authorization') != 'Bearer focus-fixture-local-only': + self.send_error(401) + return + try: + length = int(self.headers.get('Content-Length', '0')) + if not 0 < length <= 8 * 1024 * 1024: + raise ValueError('body size') + request = json.loads(self.rfile.read(length)) + if request.get('stream') is not True: + raise ValueError('streaming required') + tools = [t.get('function', {}).get('name') for t in request.get('tools', [])] + if 'attempt_completion' not in tools: + raise ValueError('native attempt_completion tool required') + except (ValueError, TypeError, AttributeError): + self.send_error(400, 'Expected streaming request with native completion tool') + return + with self.server.counter_lock: + self.server.requests += 1 + turn = self.server.requests + self.server.record(event='request', turn=turn, tools=tools) + self.send_response(200) + self.send_header('Content-Type', 'text/event-stream') + self.send_header('Cache-Control', 'no-cache') + self.end_headers() + + def emit(delta, finish=None): + chunk = {'id': f'focus-{turn}', 'object': 'chat.completion.chunk', + 'created': int(time.time()), 'model': 'focus-fixture', + 'choices': [{'index': 0, 'delta': delta, 'finish_reason': finish}]} + self.wfile.write(('data: ' + json.dumps(chunk) + '\n\n').encode()) + self.wfile.flush() + + try: + emit({'role': 'assistant', 'content': ''}) + for index in range(self.server.chunks): + emit({'content': f'History update {index + 1} for turn {turn}. '}) + time.sleep(self.server.interval) + emit({'tool_calls': [{'index': 0, 'id': f'complete-{turn}', + 'type': 'function', 'function': { + 'name': 'attempt_completion', + 'arguments': json.dumps({'result': f'Fixture turn {turn} complete.'})}}]}) + emit({}, 'tool_calls') + self.wfile.write(b'data: [DONE]\n\n') + self.wfile.flush() + self.server.record(event='completed', turn=turn, chunks=self.server.chunks) + except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError): + self.server.record(event='disconnected', turn=turn) + + +def make_server(port=19487, log=None, chunks=100, interval=0.25): + server = http.server.ThreadingHTTPServer(('127.0.0.1', port), Fixture) + server.requests = 0 + server.counter_lock = threading.Lock() + server.chunks, server.interval = chunks, interval + + def record(**fields): + if log: + with server.counter_lock, Path(log).open('a', encoding='utf-8') as output: + output.write(json.dumps({'time': time.time(), **fields}) + '\n') + server.record = record + return server + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--port', type=int, default=19487) + parser.add_argument('--log', type=Path, required=True) + args = parser.parse_args() + make_server(args.port, args.log).serve_forever() diff --git a/jetbrains_plugin/scripts/ui-harness/test_solheim_proxy.py b/jetbrains_plugin/scripts/ui-harness/test_solheim_proxy.py index 64f1113..54e857a 100644 --- a/jetbrains_plugin/scripts/ui-harness/test_solheim_proxy.py +++ b/jetbrains_plugin/scripts/ui-harness/test_solheim_proxy.py @@ -2,6 +2,7 @@ import http.client import http.server import threading +import urllib.error import unittest from unittest.mock import patch import solheim_proxy as relay @@ -21,7 +22,7 @@ def tearDown(self): def request(self, headers, body=None, method='GET'): connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) - with patch.object(relay.urllib.request, 'urlopen') as upstream: + with patch.object(relay.UPSTREAM, 'open') as upstream: connection.request(method, '/v1/models', body=body, headers=headers) response = connection.getresponse() response.read() @@ -29,6 +30,17 @@ def request(self, headers, body=None, method='GET'): upstream.assert_not_called() return response.status + def test_local_health_is_authenticated_and_does_not_contact_upstream(self): + for token, expected in [('wrong', 401), (relay.TOKEN, 200)]: + connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) + with patch.object(relay.UPSTREAM, 'open') as upstream: + connection.request('GET', '/v1/health', headers={'Authorization': 'Bearer ' + token}) + response = connection.getresponse() + self.assertEqual(expected, response.status) + response.read() + upstream.assert_not_called() + connection.close() + def test_missing_token(self): self.assertEqual(401, self.request({})) @@ -80,7 +92,7 @@ def test_authorized_request_forwards_only_upstream_key(self): response = unittest.mock.Mock(status=200, headers={'Content-Type': 'application/json'}) response.read1.side_effect = [b'{"data":[]}', b''] connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) - with patch.object(relay.urllib.request, 'urlopen', return_value=response) as upstream, patch.object(relay, 'record'): + with patch.object(relay.UPSTREAM, 'open', return_value=response) as upstream, patch.object(relay, 'record'): connection.request('GET', '/v1/models', headers={'Authorization': 'Bearer ' + relay.TOKEN}) result = connection.getresponse() self.assertEqual(200, result.status) @@ -94,7 +106,7 @@ def test_authorized_post_forwards_json_unchanged(self): response = unittest.mock.Mock(status=200, headers={'Content-Type': 'text/event-stream'}) response.read1.side_effect = [b'data: [DONE]\n', b''] connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) - with patch.object(relay.urllib.request, 'urlopen', return_value=response) as upstream, patch.object(relay, 'record'): + with patch.object(relay.UPSTREAM, 'open', return_value=response) as upstream, patch.object(relay, 'record'): connection.request('POST', '/v1/chat/completions', body=body, headers={'Authorization': 'Bearer ' + relay.TOKEN}) result = connection.getresponse() @@ -105,5 +117,40 @@ def test_authorized_post_forwards_json_unchanged(self): connection.close() +class RedirectTest(unittest.TestCase): + def test_redirects_never_reach_destination(self): + class RedirectServer(http.server.BaseHTTPRequestHandler): + paths = [] + + def log_message(self, *_): + pass + + def do_GET(self): + self.paths.append(self.path) + self.send_response(int(self.path[1:])) + self.send_header('Location', '/destination') + self.send_header('Content-Length', '0') + self.end_headers() + + server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), RedirectServer) + worker = threading.Thread(target=server.serve_forever) + worker.start() + try: + for status in [301, 302, 303, 307, 308]: + with self.subTest(status=status): + request = relay.urllib.request.Request( + 'http://127.0.0.1:' + str(server.server_port) + '/' + str(status), + headers={'Authorization': 'Bearer test-only-key'}) + with self.assertRaises(urllib.error.HTTPError) as caught: + relay.UPSTREAM.open(request, timeout=5) + self.assertEqual(status, caught.exception.code) + caught.exception.close() + self.assertEqual(['/301', '/302', '/303', '/307', '/308'], RedirectServer.paths) + finally: + server.shutdown() + server.server_close() + worker.join() + + if __name__ == '__main__': unittest.main() diff --git a/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py new file mode 100644 index 0000000..54afcec --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py @@ -0,0 +1,79 @@ +import http.client +import json +import tempfile +import threading +import time +import unittest +from pathlib import Path + +from streaming_fixture import make_server + + +class StreamingFixtureTest(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.log = Path(self.directory.name) / 'provider.jsonl' + self.server = make_server(0, self.log, chunks=3, interval=0.03) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + self.connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port, timeout=5) + + def tearDown(self): + self.connection.close() + self.server.shutdown() + self.server.server_close() + self.thread.join() + self.directory.cleanup() + + def post(self, body, token='focus-fixture-local-only'): + self.connection.request('POST', '/v1/chat/completions', json.dumps(body), + {'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json'}) + return self.connection.getresponse() + + def test_two_incremental_streams_finish_with_native_completion(self): + request = {'stream': True, 'messages': [], 'tools': [ + {'type': 'function', 'function': {'name': 'attempt_completion'}}]} + for turn in (1, 2): + started = time.monotonic() + response = self.post(request) + self.assertEqual(200, response.status) + self.assertEqual('text/event-stream', response.getheader('Content-Type')) + # Read a line before the server's delayed stream completes. + first = json.loads(response.readline().decode()[6:]) + self.assertEqual('assistant', first['choices'][0]['delta']['role']) + events = [json.loads(line[6:]) for line in response.read().decode().splitlines() + if line.startswith('data: {')] + self.assertGreaterEqual(time.monotonic() - started, 0.09) + updates = [e for e in events if e['choices'][0]['delta'].get('content')] + self.assertEqual(3, len(updates)) + tool = events[-2]['choices'][0]['delta']['tool_calls'][0] + self.assertEqual('attempt_completion', tool['function']['name']) + self.assertIn(str(turn), json.loads(tool['function']['arguments'])['result']) + self.assertEqual('tool_calls', events[-1]['choices'][0]['finish_reason']) + records = [json.loads(line) for line in self.log.read_text().splitlines()] + self.assertEqual(2, len([e for e in records if e['event'] == 'completed'])) + + def test_rejects_non_streaming_and_missing_completion_tool(self): + for request in ({'stream': False}, {'stream': True, 'tools': []}): + response = self.post(request) + self.assertEqual(400, response.status) + response.read() + self.assertEqual(0, self.server.requests) + + def test_rejects_invalid_local_token(self): + response = self.post({'stream': True}, token='wrong') + self.assertEqual(401, response.status) + response.read() + self.assertEqual(0, self.server.requests) + + def test_health_and_model_discovery(self): + self.connection.request('GET', '/health') + response = self.connection.getresponse() + self.assertEqual(b'ready', response.read()) + self.connection.request('GET', '/v1/models') + response = self.connection.getresponse() + self.assertEqual('focus-fixture', json.loads(response.read())['data'][0]['id']) + + +if __name__ == '__main__': + unittest.main() diff --git a/jetbrains_plugin/src/main/kotlin/org/zoocode/jetbrains/editor/EditorAndDocManager.kt b/jetbrains_plugin/src/main/kotlin/org/zoocode/jetbrains/editor/EditorAndDocManager.kt index 4a29cd0..87b3ace 100644 --- a/jetbrains_plugin/src/main/kotlin/org/zoocode/jetbrains/editor/EditorAndDocManager.kt +++ b/jetbrains_plugin/src/main/kotlin/org/zoocode/jetbrains/editor/EditorAndDocManager.kt @@ -24,6 +24,8 @@ import com.intellij.openapi.fileEditor.FileDocumentManager import com.intellij.openapi.fileEditor.FileEditor import com.intellij.openapi.fileEditor.FileEditorManager import com.intellij.openapi.fileEditor.FileEditorManagerListener +import com.intellij.openapi.fileEditor.ex.FileEditorManagerEx +import com.intellij.openapi.fileEditor.impl.FileEditorOpenOptions import com.intellij.openapi.fileTypes.FileType import com.intellij.openapi.project.Project import com.intellij.openapi.vfs.LocalFileSystem @@ -208,6 +210,21 @@ class EditorAndDocManager(val project: Project) : Disposable { } + private suspend fun openFilePreservingFocus(file: VirtualFile): Array { + if (project.isDisposed) throw CancellationException("Project disposed before editor open") + // Synchronous editor loading pumps a restricted event loop that can + // discard chat keys, even when no focus transfer is requested. + // Invoke the JVM no-arg constructor: Kotlin's default-argument constructor + // changes signature between supported IDE builds. + val openOptions = FileEditorOpenOptions::class.java.getConstructor() + .newInstance().withRequestFocus(false) + val editors = FileEditorManagerEx.getInstanceEx(project) + .openFile(file, openOptions).allEditors.toTypedArray() + // Suspension can outlive the project; do not create handles or tabs afterward. + if (project.isDisposed) throw CancellationException("Project disposed during editor open") + return editors + } + /** * Opens a file editor for the document. * @@ -223,14 +240,20 @@ class EditorAndDocManager(val project: Project) : Disposable { suspend fun openEditor(documentUri: URI, preserveFocus: Boolean = false, options: ResolvedTextEditorConfiguration = ResolvedTextEditorConfiguration()): EditorHolder { val fileEditorManager = FileEditorManager.getInstance(project) val path = documentUri.path - var ideaEditor : Array? = null + var ideaEditor : Array? = null val vfs = LocalFileSystem.getInstance() val file = vfs.findFileByPath(path) file?.let { - ApplicationManager.getApplication().invokeAndWait { - if (!project.isDisposed) { - ideaEditor = fileEditorManager.openFile(it, editorFocusRequest(preserveFocus)) + if (!project.isDisposed) { + if (preserveFocus) { + ideaEditor = openFilePreservingFocus(it) + } else { + ApplicationManager.getApplication().invokeAndWait { + if (!project.isDisposed) { + ideaEditor = fileEditorManager.openFile(it, editorFocusRequest(preserveFocus)) + } + } } } } @@ -272,16 +295,16 @@ class EditorAndDocManager(val project: Project) : Disposable { if (content1 != null && content2 != null){ val request = SimpleDiffRequest(title, content1, content2, left.path, documentUri.path) var ideaEditor : Array? = null - ApplicationManager.getApplication().invokeAndWait{ - if (!project.isDisposed) { - val requestChain: DiffRequestChain = SimpleDiffRequestChain(request) - val diffFile = ChainDiffVirtualFile(requestChain, DiffBundle.message("label.default.diff.editor.tab.name", *arrayOfNulls(0))) - when (diffOpenPlan(preserveFocus)) { - DiffOpenPlan.PRESERVE_VIA_FILE_EDITOR -> { - ideaEditor = FileEditorManager.getInstance(project) - .openFile(diffFile, editorFocusRequest(preserveFocus)) - } - DiffOpenPlan.DELIBERATE_VIA_DIFF_HELPER -> { + if (!project.isDisposed) { + val requestChain: DiffRequestChain = SimpleDiffRequestChain(request) + val diffFile = ChainDiffVirtualFile(requestChain, DiffBundle.message("label.default.diff.editor.tab.name", *arrayOfNulls(0))) + when (diffOpenPlan(preserveFocus)) { + DiffOpenPlan.PRESERVE_VIA_FILE_EDITOR -> { + ideaEditor = openFilePreservingFocus(diffFile) + } + DiffOpenPlan.DELIBERATE_VIA_DIFF_HELPER -> { + ApplicationManager.getApplication().invokeAndWait { + if (project.isDisposed) return@invokeAndWait LocalFileSystem.getInstance().findFileByPath(documentUri.path) ?.let { ApplicationManager.getApplication().runReadAction { FileEditorManager.getInstance(project).closeFile(it) } diff --git a/jetbrains_plugin/src/main/kotlin/org/zoocode/jetbrains/editor/EditorCommands.kt b/jetbrains_plugin/src/main/kotlin/org/zoocode/jetbrains/editor/EditorCommands.kt index fb4601d..b4fba5a 100644 --- a/jetbrains_plugin/src/main/kotlin/org/zoocode/jetbrains/editor/EditorCommands.kt +++ b/jetbrains_plugin/src/main/kotlin/org/zoocode/jetbrains/editor/EditorCommands.kt @@ -150,8 +150,8 @@ fun decodeDiffPreserveFocus(columnOrOptions: Any?): Boolean = decodePreserveFocu /** * Maps the preservation intent to the platform focus boolean. The * [FileEditorManager.openFile] boolean requests editor focus, so a preserving - * open must request the inverse. Both open calls in [EditorAndDocManager] pass - * through this mapping. + * open must request the inverse. Deliberate file opens use this mapping; preserving + * file and diff opens use the suspending API with an explicit false focus option. */ fun editorFocusRequest(preserveFocus: Boolean): Boolean = !preserveFocus diff --git a/jetbrains_plugin/src/test/kotlin/org/zoocode/jetbrains/editor/EditorFocusIntentTest.kt b/jetbrains_plugin/src/test/kotlin/org/zoocode/jetbrains/editor/EditorFocusIntentTest.kt index e5d9304..a0d7318 100644 --- a/jetbrains_plugin/src/test/kotlin/org/zoocode/jetbrains/editor/EditorFocusIntentTest.kt +++ b/jetbrains_plugin/src/test/kotlin/org/zoocode/jetbrains/editor/EditorFocusIntentTest.kt @@ -11,8 +11,8 @@ import org.junit.Test /** * Covers the display-intent decoding shared by tryShowTextDocument and - * workbench_diff, the inverse mapping that both [EditorAndDocManager] open - * calls pass to FileEditorManager.openFile, and the diff routing decision. + * workbench_diff, the inverse platform focus mapping, and the diff routing + * decision. Preserving file and diff opens share the suspending platform API. * The real open behavior is exercised live by the UI harness, which needs a * real IDE. */