diff --git a/.github/workflows/windows-chat-focus.yml b/.github/workflows/windows-chat-focus.yml index 7d3136e..8fd14bc 100644 --- a/.github/workflows/windows-chat-focus.yml +++ b/.github/workflows/windows-chat-focus.yml @@ -27,6 +27,12 @@ on: permissions: contents: read +env: + # Reproduce the release under investigation with the current Kotlin changes. + # The build job packages this extension version. The test steps check that the + # plugin ZIP really contains it, because the ZIP name does not set it. + ZOO_EXTENSION_VERSION: "3.96.0" + concurrency: # Skipped label events must not replace a pending build for this PR. group: windows-chat-focus-${{ github.ref }}-${{ github.event.action == 'labeled' && github.event.label.name != 'live-solheim' && 'ignored-label' || 'validation' }} @@ -40,8 +46,6 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 90 env: - # Reproduce the release under investigation with the current Kotlin changes. - ZOO_EXTENSION_VERSION: "3.96.0" # This job packages the plugin; headful tests use the IDE's bundled JCEF. PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" steps: @@ -65,7 +69,7 @@ jobs: sudo apt-get install --no-install-recommends build-essential libx11-dev libxkbfile-dev pkg-config - name: Validate the local streaming fixture and authenticated relay working-directory: jetbrains_plugin/scripts/ui-harness - run: python3 -m unittest -v test_streaming_fixture.py test_solheim_proxy.py + run: python3 -m unittest -v test_streaming_fixture.py test_solheim_proxy.py test_check_history_store.py - name: Prepare extension host source run: ./scripts/setup.sh --skip-deps --verbose - name: Cache the cross-platform VS Code runtime @@ -82,10 +86,10 @@ jobs: if-no-files-found: error windows-e2e: - name: Native typing and streaming chat (IDEA ${{ matrix.ide }}) + name: Native typing, streaming chat, and shared history (IDEA ${{ matrix.ide }}) needs: build runs-on: windows-2022 - timeout-minutes: 30 + timeout-minutes: 60 strategy: fail-fast: false matrix: @@ -104,8 +108,10 @@ jobs: - uses: actions/setup-python@v6 with: python-version: "3.12" - - name: Validate fixture HTTP behavior on Windows - run: python -m unittest discover -s jetbrains_plugin/scripts/ui-harness -p test_streaming_fixture.py -v + - name: Validate fixture HTTP behavior and the history store checker on Windows + run: | + python -m unittest discover -s jetbrains_plugin/scripts/ui-harness -p test_streaming_fixture.py -v + python -m unittest discover -s jetbrains_plugin/scripts/ui-harness -p test_check_history_store.py -v - uses: actions/download-artifact@v4 with: name: chat-focus-plugin @@ -138,7 +144,22 @@ jobs: if ($plugins.Count -ne 1) { throw 'Expected exactly one plugin distribution' } & ./jetbrains_plugin/scripts/ui-harness/run-windows.ps1 ` -IdeDir $env:FOCUS_IDE_DIR -PluginZip $plugins[0].FullName ` + -ExpectExtensionVersion $env:ZOO_EXTENSION_VERSION ` -EvidenceDir "$env:GITHUB_WORKSPACE/focus-evidence" + # Issue #6. Two IDEs share ~/.zoo-code. Each one runs a parent task that + # delegates to a subtask. The step fails if the shared history store has a + # shared index file, invalid items, wrong parent/child links, more than three + # task switches in the webview, or an invalid status transition in the IDE logs. + # It does not detect a view that changes to the parent during the subtask and stays there. + - name: Run two IDEs with parent tasks and subtasks against one history store + if: ${{ !cancelled() }} + run: | + $plugins = @(Get-ChildItem focus-plugin -Filter *.zip) + if ($plugins.Count -ne 1) { throw 'Expected exactly one plugin distribution' } + & ./jetbrains_plugin/scripts/ui-harness/run-windows.ps1 -HistoryRace ` + -IdeDir $env:FOCUS_IDE_DIR -PluginZip $plugins[0].FullName ` + -ExpectExtensionVersion $env:ZOO_EXTENSION_VERSION ` + -EvidenceDir "$env:GITHUB_WORKSPACE/history-evidence" -DeadlineSeconds 900 - name: Upload focus traces, screenshots, and run identity if: always() uses: actions/upload-artifact@v4 @@ -146,6 +167,13 @@ jobs: name: windows-chat-focus-${{ matrix.ide }}-${{ github.sha }} path: focus-evidence/ if-no-files-found: error + - name: Upload shared history evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: windows-history-race-${{ matrix.ide }}-${{ github.sha }} + path: history-evidence/ + if-no-files-found: warn live-solheim: name: Live Solheim terminal and diff focus (WebStorm 2026.2.1) @@ -201,6 +229,7 @@ jobs: $env:SOLHEIM_API_KEY = $liveKey & ./jetbrains_plugin/scripts/ui-harness/run-windows.ps1 ` -IdeDir $env:FOCUS_IDE_DIR -PluginZip $plugins[0].FullName ` + -ExpectExtensionVersion $env:ZOO_EXTENSION_VERSION ` -Provider solheim -DeadlineSeconds 780 -RequireWindows11:($env:REQUIRE_WINDOWS11 -eq 'true') ` -EvidenceDir "$env:GITHUB_WORKSPACE/live-evidence/run-$iteration" } diff --git a/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java b/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java index f03eb2e..6e8b544 100644 --- a/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java +++ b/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java @@ -49,6 +49,10 @@ public final class HarnessDriver implements AppLifecycleListener { private final java.util.Set approvedCommands = new java.util.HashSet<>(); private final boolean fixtureMode = Boolean.getBoolean("zoo.harness.fixture"); private final boolean diffStress = Boolean.getBoolean("zoo.harness.diffStress"); + // Issue #6: two IDEs share one history store. This mode submits one parent task that + // delegates to a subtask, then waits for the parent to resume. It does no focus checks. + private final boolean historyRace = Boolean.getBoolean("zoo.harness.historyRace"); + private final String historyTag = System.getProperty("zoo.harness.historyTag", "A"); private final boolean windowsLive = !fixtureMode && System.getProperty("os.name", "").startsWith("Windows"); private String windowsCommand(String step) { @@ -248,7 +252,7 @@ private void mark(String step) { // Manual provider-backed battery: real React textarea, real extension tools. private JsonObject realSnapshot(Object webview, AtomicReference title) throws Exception { int id=++realSequence; - String script="(()=>{const snapshot=JSON.stringify((()=>{let e=[...document.querySelectorAll('textarea')].find(x=>x.offsetWidth&&x.offsetHeight);let r=e&&e.getBoundingClientRect();let z=window.__focusE2E||{};let m=z.messages||[];let last=m[m.length-1]||{};return {snapshot:"+id+",input:!!e,value:e?e.value:null,active:e===document.activeElement,windowFocused:document.hasFocus(),disabled:e?e.disabled:null,rect:r?{x:r.x,y:r.y,w:r.width,h:r.height}:null,messages:m.length,events:z.events||0,partial:z.partial||0,lastAsk:last.ask||null,lastTs:last.ts||null,lastText:typeof last.text==='string'?last.text.slice(0,300):null,lastSay:last.say||null,lastPartial:!!last.partial,provider:z.provider||null,body:document.body.innerText.slice(-1200)};})());console.log('ZFE2E:'+snapshot);document.title='ZFE2E:'+snapshot;})();"; + String script="(()=>{const snapshot=JSON.stringify((()=>{let e=[...document.querySelectorAll('textarea')].find(x=>x.offsetWidth&&x.offsetHeight);let r=e&&e.getBoundingClientRect();let z=window.__focusE2E||{};let m=z.messages||[];let last=m[m.length-1]||{};return {snapshot:"+id+",input:!!e,value:e?e.value:null,active:e===document.activeElement,windowFocused:document.hasFocus(),disabled:e?e.disabled:null,rect:r?{x:r.x,y:r.y,w:r.width,h:r.height}:null,messages:m.length,events:z.events||0,partial:z.partial||0,taskId:z.taskId||null,tasks:z.tasks||[],lastAsk:last.ask||null,lastTs:last.ts||null,lastText:typeof last.text==='string'?last.text.slice(0,300):null,lastSay:last.say||null,lastPartial:!!last.partial,provider:z.provider||null,body:document.body.innerText.slice(-1200)};})());console.log('ZFE2E:'+snapshot);document.title='ZFE2E:'+snapshot;})();"; runOnEdt(() -> invoke(webview, "executeJavaScript", script)); long resend=System.currentTimeMillis()+1000; long deadline=System.currentTimeMillis()+30000; @@ -318,7 +322,7 @@ private void runRealConversation(Project project) throws Exception { if (!addTitleHandler(browser, invoke(browser, "getCefBrowser"), title)) { throw new IllegalStateException("Cannot register JCEF snapshot handler; see IDE log for cause"); } - invoke(webview, "executeJavaScript", "window.__focusE2E={messages:[],events:0,partial:0};window.addEventListener('message',e=>{let d=e.data,z=window.__focusE2E;if(!d)return;z.events++;if(d.type==='state'&&d.state){if(d.state.clineMessages)z.messages=d.state.clineMessages.slice();z.provider=d.state.apiConfiguration&&d.state.apiConfiguration.apiProvider;}if(d.type==='messageUpdated'||d.type==='partialMessage'){let m=d.clineMessage||d.partialMessage;if(m){if(m.partial)z.partial++;let i=z.messages.findIndex(x=>x.ts===m.ts);if(i>=0)z.messages[i]=m;else z.messages.push(m);}}});"); + invoke(webview, "executeJavaScript", "window.__focusE2E={messages:[],events:0,partial:0};window.addEventListener('message',e=>{let d=e.data,z=window.__focusE2E;if(!d)return;z.events++;if(d.type==='state'&&d.state){if(d.state.clineMessages)z.messages=d.state.clineMessages.slice();z.taskId=d.state.currentTaskItem?d.state.currentTaskItem.id:null;(z.tasks=z.tasks||[]);if(z.taskId&&z.tasks[z.tasks.length-1]!==z.taskId)z.tasks.push(z.taskId);z.provider=d.state.apiConfiguration&&d.state.apiConfiguration.apiProvider;}if(d.type==='messageUpdated'||d.type==='partialMessage'){let m=d.clineMessage||d.partialMessage;if(m){if(m.partial)z.partial++;let i=z.messages.findIndex(x=>x.ts===m.ts);if(i>=0)z.messages[i]=m;else z.messages.push(m);}}});"); invoke(webview,"executeJavaScript","window.__focusNodeSeq=0;function e2eNode(e){if(!e)return null;if(!e.__e2eid)e.__e2eid=++window.__focusNodeSeq;return e.tagName+'@'+e.__e2eid;}for(let kind of ['keydown','input','focusin','focusout'])document.addEventListener(kind,e=>{if(kind==='keydown'&&!['x','y'].includes(e.key))return;console.info('E2EKEY:'+JSON.stringify({kind,ts:Date.now(),key:e.key||null,mod:e.ctrlKey||e.altKey||e.metaKey||false,node:e2eNode(e.target),active:e2eNode(document.activeElement),hasFocus:document.hasFocus(),length:typeof e.target.value==='string'?e.target.value.length:null}));},true);for(let kind of ['focus','blur'])window.addEventListener(kind,e=>console.info('E2EWINDOW:'+JSON.stringify({kind,ts:Date.now(),active:e2eNode(document.activeElement),hasFocus:document.hasFocus()})),true);"); realSend(webview, "{\"type\":\"webviewDidLaunch\"}"); realSend(webview, "{\"type\":\"mode\",\"text\":\"code\"}"); @@ -335,6 +339,7 @@ private void runRealConversation(Project project) throws Exception { realSend(webview, config.toString()); realSend(webview, "{\"type\":\"updateSettings\",\"updatedSettings\":{\"autoApprovalEnabled\":true,\"alwaysAllowReadOnly\":true,\"alwaysAllowWrite\":true,\"alwaysAllowExecute\":true,\"allowedCommands\":[\"printf\",\"sleep\",\"cat\",\"pwd\"],\"terminalShellIntegrationDisabled\":false,\"destructiveCommandGuardEnabled\":false,\"maxRequests\":20,\"telemetrySetting\":\"disabled\"}}"); if (!fixtureMode) realSend(webview, "{\"type\":\"updateSettings\",\"updatedSettings\":{\"alwaysAllowExecute\":false}}"); + if (historyRace) realSend(webview, "{\"type\":\"updateSettings\",\"updatedSettings\":{\"alwaysAllowSubtasks\":true}}"); realSend(webview, "{\"type\":\"didShowAnnouncement\"}"); invoke(webview, "executeJavaScript", "[...document.querySelectorAll('button')].find(b=>b.textContent.trim()==='Close')?.click();"); JsonObject snap = null; @@ -351,6 +356,21 @@ private void runRealConversation(Project project) throws Exception { if (!Boolean.TRUE.equals(ready)) throw new IllegalStateException("Chat remained hidden during indexing"); snap = realSnapshot(webview,title); screenshot("p-configured"); + if (historyRace) { + // Start the task with a webview message, not the keyboard. Two IDEs run together, + // and a background IDE window does not get native focus on Windows. + realSend(webview, "{\"type\":\"newTask\",\"text\":\"HISTORY_RACE_PARENT " + historyTag + + " delegate one subtask, then finish.\"}"); + boolean started = false; + for (int i = 0; i < 60 && !started; i++) { + Thread.sleep(1000); + started = realSnapshot(webview, title).get("messages").getAsInt() > 0; + } + if (!started) { fail("realTaskMissing", "No task appeared after the newTask message"); return; } + mark("P-real-conversation-submitted"); + waitForHistoryRace(webview, title); + return; + } if (diffStress) checkDeliberateEditorFocus(project); // Establish native focus before typing anything. A freshly mounted OSR // surface may paint after its DOM is ready. Setup clicks are bounded; @@ -503,6 +523,46 @@ private void runRealConversation(Project project) throws Exception { finish(fixtureMode ? "testP-streaming-fixture" : "testP-real-solheim-conversation", firstCompleted&&completed&&commandsCovered&&typingHealthy&&focusHealthy&&firstStreamed&&followupStreamed&&historyGrew&&fixtureChanged?"PASS":"FAIL-REAL-CONVERSATION"); } + /** + * Waits until the parent task resumes and completes after its subtask. The webview shows + * one task at a time. The expected task order is parent, child, parent. A fourth change + * of task fails the test. The test counts task switches only. It cannot see a view + * that changes to the parent during the subtask and stays there until the parent resumes. + */ + private void waitForHistoryRace(Object webview, AtomicReference title) throws Exception { + long deadline = System.currentTimeMillis() + 420_000; + List sequence = new ArrayList<>(); + while (System.currentTimeMillis() < deadline) { + JsonObject snap = realSnapshot(webview, title); + // The page records every task id in arrival order. Polling alone can miss the + // parent, because a delegation can finish before the first poll. + List seen = new ArrayList<>(); + snap.getAsJsonArray("tasks").forEach(id -> seen.add(id.getAsString())); + if (!seen.equals(sequence)) { + sequence = seen; + record("H_taskSequence_" + sequence.size(), sequence); + } + String ask = snap.get("lastAsk").isJsonNull() ? "" : snap.get("lastAsk").getAsString(); + if (!answerCommandAsk(webview, snap, ask)) return; + if (sequence.size() > 3) { + fail("subtaskViewFlipped", "Task view changed " + sequence.size() + " times: " + sequence); + return; + } + String current = snap.get("taskId").isJsonNull() ? null : snap.get("taskId").getAsString(); + boolean done = sequence.size() == 3 && sequence.get(2).equals(sequence.get(0)) + && sequence.get(0).equals(current) && "completion_result".equals(ask) + && !snap.get("lastPartial").getAsBoolean(); + if (done) { + // Give the extension time to persist the final history items. + Thread.sleep(5_000); + finish("testH-history-race-" + historyTag, "PASS"); + return; + } + Thread.sleep(1_000); + } + fail("historyRaceTimeout", "Parent did not resume after its subtask; task order " + sequence); + } + private void clickInBrowser(Object jbCefBrowser, JsonObject rect) throws Exception { final Point[] loc = new Point[1]; runOnEdt(() -> { diff --git a/jetbrains_plugin/scripts/ui-harness/README.md b/jetbrains_plugin/scripts/ui-harness/README.md index 0134c4c..5eeb3b0 100644 --- a/jetbrains_plugin/scripts/ui-harness/README.md +++ b/jetbrains_plugin/scripts/ui-harness/README.md @@ -42,6 +42,40 @@ it has no input for an existing release ZIP. Its IDEA matrix is 2026.1.5.1 and 2026.2.1, so those jobs do not establish compatibility with IDEA 2023.3.8. Record the exact ZIP, IDE version, and evidence directory when comparing builds. +## Shared history race test (issue #6) + +`run-windows.ps1 -HistoryRace` starts two IDEs with separate profiles and the same +home directory, so both use one `~/.zoo-code/globalStorage`. Both IDEs start together. +Each IDE starts a parent task with a webview `newTask` message. The test uses no keyboard, +because a background IDE window gets no native focus on Windows. The fixture makes the +parent call `new_task`. It holds both parents at a barrier so the delegations overlap. +It then streams both child tasks and lets each parent resume and finish. `check_history_store.py` then checks the store: + +1. No `_index.json` was written. +2. Every `history_item.json` is valid, has a real title, and has `size` above zero. +3. Each parent lists only its own child, and each child names only its own parent. +4. Each child is `completed` and no parent still waits on a child. +5. The provider log shows both barriers released, no disconnect, and six completed streams. +6. The IDE logs hold no invalid status transition, rejected delta, failed persist, or re-delegate error message. + +The driver also fails if the webview changes task more than three times. The expected +order is parent, child, parent. The test counts task switches only. It does not detect a +view that changes to the parent during the subtask and stays there until the parent resumes. + +The run uses the real user profile store, so the script refuses to start unless +`GITHUB_ACTIONS` is `true`. The Windows CI job has a 60 minute timeout and uploads +the evidence as `windows-history-race--`. + +`-ExpectExtensionVersion` (`run-windows.ps1`) makes a run fail when the Zoo Code extension +in the plugin ZIP has another version. CI passes it to the chat focus, shared history, +and live Solheim steps. + +`streaming_fixture.py` flags: `--port`, `--log` (required), `--scenario history-race`, +`--chunks`, `--interval` (seconds between chunks), `--barrier` (parents to hold). + +The unit tests run without an IDE. Run them from this directory: +`python3 -m unittest test_streaming_fixture test_check_history_store`. + ## Linux live setup Store the Solheim key in a private file outside the repository (mode 0600). diff --git a/jetbrains_plugin/scripts/ui-harness/check_history_store.py b/jetbrains_plugin/scripts/ui-harness/check_history_store.py new file mode 100644 index 0000000..b76de99 --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/check_history_store.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Check the shared Zoo Code task history after a concurrent two-IDE run. + +Issue #6: several IDE processes write one global history store. A safe store +keeps one valid history_item.json per task, with no shared index file and with +parent and child links that stay inside one IDE's task tree. +""" +import argparse +import json +import re +import sys +from pathlib import Path + +PLACEHOLDER = re.compile(r'^\s*(Task|Work) #\d+') +BAD_LOG = re.compile(r'invalid status transition|DeltaRejectedError|failed to persist|Cannot re-delegate', re.I) +MARKER = re.compile(r'HISTORY_RACE_(PARENT|CHILD) (\w+)') + + +def load_items(storage, since): + """Return ({task id: item}, failures) for tasks written at or after `since`.""" + items, failures = {}, [] + for index in storage.glob('**/tasks/_index.json'): + if index.stat().st_mtime >= since: + failures.append(f'shared index file was written: {index}') + for path in sorted(storage.glob('**/tasks/*/history_item.json')): + if path.stat().st_mtime < since: + continue + try: + item = json.loads(path.read_text(encoding='utf-8')) + except (OSError, ValueError) as error: + failures.append(f'{path.parent.name}: history_item.json is not valid JSON ({error})') + continue + if not isinstance(item, dict): + failures.append(f'{path.parent.name}: history_item.json is not an object') + continue + if item.get('id') != path.parent.name: + failures.append(f'{path.parent.name}: id field is {item.get("id")!r}') + continue + items[item['id']] = item + return items, failures + + +def check_items(items, instances): + failures = [] + tasks = {} + for task_id, item in items.items(): + text = item.get('task') or '' + if PLACEHOLDER.match(text) or not text.strip(): + failures.append(f'{task_id}: placeholder or empty title {text!r}') + continue + match = MARKER.search(text) + if not match: + failures.append(f'{task_id}: title does not carry a run marker: {text!r}') + continue + if not item.get('size'): + failures.append(f'{task_id}: size is {item.get("size")!r} after real work') + tasks[task_id] = (match.group(1), match.group(2), item) + parents = {i: t for i, t in tasks.items() if t[0] == 'PARENT'} + children = {i: t for i, t in tasks.items() if t[0] == 'CHILD'} + if len(parents) != instances or len(children) != instances: + failures.append(f'expected {instances} parents and {instances} children, ' + f'found {len(parents)} and {len(children)}') + for parent_id, (_, tag, parent) in parents.items(): + own = [c for c, t in children.items() if t[1] == tag] + if len(own) != 1: + failures.append(f'{parent_id}: expected one child for tag {tag}, found {len(own)}') + continue + child_id = own[0] + child = children[child_id][2] + if parent.get('childIds') != [child_id]: + failures.append(f'{parent_id}: childIds {parent.get("childIds")!r}, expected {[child_id]!r}') + if child.get('parentTaskId') != parent_id: + failures.append(f'{child_id}: parentTaskId {child.get("parentTaskId")!r}, expected {parent_id!r}') + if child.get('status') != 'completed': + failures.append(f'{child_id}: child status {child.get("status")!r}, expected completed') + if parent.get('status') == 'delegated' or parent.get('awaitingChildId') or parent.get('delegatedToId'): + failures.append(f'{parent_id}: parent still waits on a child (status {parent.get("status")!r})') + if parent.get('workspace') != child.get('workspace'): + failures.append(f'{parent_id}: parent and child workspaces differ') + workspaces = {t[2].get('workspace') for t in parents.values()} + if instances > 1 and len(workspaces) != instances: + failures.append(f'expected {instances} distinct workspaces, found {sorted(map(str, workspaces))}') + return failures + + +def check_provider(path, instances): + events, failures = [], [] + for number, line in enumerate(path.read_text(encoding='utf-8').splitlines(), 1): + if not line.strip(): + continue + try: + events.append(json.loads(line)) + except ValueError as error: + failures.append(f'{path.name}:{number}: provider log line is not valid JSON ({error})') + for event in events: + if not isinstance(event, dict): + failures.append(f'{path.name}: provider log line is not an object: {event!r}') + names = [e.get('event') for e in events if isinstance(e, dict)] + if names.count('barrier-released') != instances: + failures.append(f'barrier released {names.count("barrier-released")} times, expected {instances}') + for bad in ('barrier-timeout', 'disconnected'): + if bad in names: + failures.append(f'provider logged {bad}') + if names.count('completed') != 3 * instances: + failures.append(f'{names.count("completed")} completed streams, expected {3 * instances}') + return failures + + +def check_logs(paths): + failures = [] + for path in paths: + if not path.exists(): + failures.append(f'IDE log does not exist: {path}') + continue + for number, line in enumerate(path.read_text(encoding='utf-8', errors='replace').splitlines(), 1): + if BAD_LOG.search(line): + failures.append(f'{path.name}:{number}: {line.strip()[:200]}') + return failures + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--storage', type=Path, required=True) + parser.add_argument('--since', type=float, required=True, help='epoch seconds when the run started') + parser.add_argument('--provider-log', type=Path, required=True) + parser.add_argument('--ide-log', type=Path, action='append', required=True) + parser.add_argument('--instances', type=int, default=2) + parser.add_argument('--report', type=Path) + args = parser.parse_args() + items, failures = load_items(args.storage, args.since) + failures += check_items(items, args.instances) + failures += check_provider(args.provider_log, args.instances) + failures += check_logs(args.ide_log) + report = {'tasks': len(items), 'failures': failures} + text = json.dumps(report, indent=2) + if args.report: + args.report.write_text(text, encoding='utf-8') + print(text) + return 1 if failures else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 b/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 index f7c4cf8..b64b81d 100644 --- a/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 +++ b/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 @@ -6,19 +6,30 @@ param( [Parameter(Mandatory = $true)][string]$EvidenceDir, [ValidateRange(60, 1800)][int]$DeadlineSeconds = 600, [ValidateSet('fixture', 'solheim')][string]$Provider = 'fixture', - [switch]$RequireWindows11 + [switch]$RequireWindows11, + # Fail when the Zoo Code extension inside the plugin ZIP has a different version. + # The ZIP name does not set that version. + [string]$ExpectExtensionVersion, + # Issue #6: run two IDEs against one history store, each with a parent task and a subtask. + [switch]$HistoryRace ) $ErrorActionPreference = 'Stop' $IdeDir = (Resolve-Path $IdeDir).Path $PluginZip = (Resolve-Path $PluginZip).Path $EvidenceDir = [IO.Path]::GetFullPath($EvidenceDir) +if ($HistoryRace -and $env:GITHUB_ACTIONS -ne 'true') { + # This mode shares the real user profile store. Copies of its tasks go to the evidence. + throw 'The history race test uses the real user profile. Run it only on a disposable CI runner.' +} New-Item -ItemType Directory -Force $EvidenceDir | Out-Null if (Test-Path (Join-Path $EvidenceDir 'status.txt')) { throw 'Use a fresh evidence directory for each run.' } $runDir = Join-Path ([IO.Path]::GetTempPath()) ('zoo-focus-' + [guid]::NewGuid().ToString('N')) New-Item -ItemType Directory $runDir | Out-Null $ideProcess = $null +$instances = @{} $fixtureProcess = $null $live = $Provider -eq 'solheim' +if ($HistoryRace -and $live) { throw 'The history race test uses only the local fixture provider.' } $originalRelayToken = $env:SOLHEIM_RELAY_TOKEN $originalKeyFile = $env:SOLHEIM_KEY_FILE $originalProxyLog = $env:SOLHEIM_PROXY_LOG @@ -92,6 +103,12 @@ try { if (-not (Test-Path "$plugins/ZooCode/runtime/extension.js")) { throw 'Distribution must contain the release extension-host runtime; build with debugMode=release.' } + $extensionVersion = (Get-Content "$plugins/ZooCode/zoo-code/package.json" -Raw | ConvertFrom-Json).version + Set-Content (Join-Path $EvidenceDir 'extension-version.txt') $extensionVersion + Write-Output "Bundled Zoo Code extension version: $extensionVersion" + if ($ExpectExtensionVersion -and $extensionVersion -ne $ExpectExtensionVersion) { + throw "Plugin ZIP bundles Zoo Code extension $extensionVersion, expected $ExpectExtensionVersion." + } $project = Join-Path $runDir 'workspace' New-Item -ItemType Directory -Force "$project/src" | Out-Null 'public class Main { public static void main(String[] args) { System.out.println("throwaway"); } }' | Set-Content "$project/src/Main.java" @@ -142,6 +159,7 @@ idea.plugins.path=$profilePath/plugins # Keep the provider alive without creating a foreground test obstacle. $providerScript = "$PSScriptRoot/streaming_fixture.py" $providerArguments = @("`"$providerScript`"", '--log', "`"$EvidenceDir/provider.jsonl`"") + if ($HistoryRace) { $providerArguments += @('--scenario', 'history-race', '--chunks', '200') } $readyUri = 'http://127.0.0.1:19487/health' $readyHeaders = @{} if ($live) { @@ -177,6 +195,87 @@ idea.plugins.path=$profilePath/plugins $env:SOLHEIM_KEY_FILE = $null $env:SOLHEIM_PROXY_LOG = $null } + if ($HistoryRace) { + # Both IDEs keep their own config, system, and log paths. They share the plugins + # directory, the home directory, and so ~/.zoo-code/globalStorage, as in issue #6. + $storage = Join-Path $env:USERPROFILE '.zoo-code/globalStorage' + $since = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() - 2 + $product = Get-Content "$IdeDir/product-info.json" -Raw | ConvertFrom-Json + $launcher = if ($product.productCode -eq 'WS') { 'webstorm64.exe' } else { 'idea64.exe' } + function Start-RaceIde([string]$tag) { + $profileDir = Join-Path $runDir "ide-$tag" + $workspace = Join-Path $runDir "workspace-$tag" + New-Item -ItemType Directory -Force "$workspace/src" | Out-Null + Copy-Item "$project/src/Main.java" "$workspace/src/Main.java" + $evidence = Join-Path $EvidenceDir $tag + New-Item -ItemType Directory -Force $evidence | Out-Null + $profilePath = $profileDir.Replace('\', '/') + $pluginsPath = $plugins.Replace('\', '/') + $propertiesFile = Join-Path $runDir "idea-$tag.properties" + @" +idea.config.path=$profilePath/config +idea.system.path=$profilePath/system +idea.log.path=$profilePath/log +idea.plugins.path=$pluginsPath +"@ | Set-Content $propertiesFile + $optionsFile = Join-Path $runDir "idea-$tag.vmoptions" + @" +-Xmx1024m +-XX:ActiveProcessorCount=2 +-Didea.trust.all.projects=true +-Didea.initially.ask.config=false +-Djb.consents.confirmation.enabled=false +-Djb.privacy.policy.text= +-Dide.show.tips.on.startup.default.value=false +-Dzoo.harness.mode=P +-Dzoo.harness.fixture=true +-Dzoo.harness.historyRace=true +-Dzoo.harness.historyTag=$tag +-Dzoo.harness.evidence=$($evidence.Replace('\', '/')) +"@ | Set-Content $optionsFile + $env:IDEA_PROPERTIES = $propertiesFile + $env:IDEA_VM_OPTIONS = $optionsFile + $env:IDEA64_VM_OPTIONS = $optionsFile + $env:WEBIDE_PROPERTIES = $propertiesFile + $env:WEBIDE_VM_OPTIONS = $optionsFile + $process = Start-Process -FilePath "$IdeDir/bin/$launcher" -ArgumentList "`"$workspace`"" -PassThru ` + -RedirectStandardOutput "$evidence/launcher-stdout.log" -RedirectStandardError "$evidence/launcher-stderr.log" + $instances[$tag] = [pscustomobject]@{ Process = $process; Evidence = $evidence; Log = "$profileDir/log" } + } + function Wait-RaceIde([string]$tag, [scriptblock]$until, [string]$what, [int]$seconds) { + $instance = $instances[$tag] + $limit = [DateTime]::UtcNow.AddSeconds($seconds) + while ([DateTime]::UtcNow -lt $limit) { + if (& $until $instance) { return } + if ($instance.Process.HasExited) { throw "IDE $tag exited before ${what}: $($instance.Process.ExitCode)" } + Start-Sleep -Seconds 1 + } + throw "IDE $tag did not reach $what within $seconds seconds" + } + # The driver starts each task with a webview message and uses no keyboard. A background + # IDE window gets no native focus on Windows, so keyboard input would fail for IDE B. + # A waits at the provider barrier until B submits, so both delegations start together. + Start-RaceIde 'A' + Start-RaceIde 'B' + foreach ($tag in 'A', 'B') { + Wait-RaceIde $tag { param($i) Test-Path "$($i.Evidence)/status.txt" } 'a result' $DeadlineSeconds + $status = (Get-Content "$($instances[$tag].Evidence)/status.txt" -Raw).Trim() + if ($status -ne 'PASS') { throw "History race IDE ${tag}: $status. See $tag/evidence.json and $tag/progress.log" } + } + # Keep a copy of the store that this run wrote, for the evidence archive. + $copy = Join-Path $EvidenceDir 'history-store' + New-Item -ItemType Directory -Force $copy | Out-Null + Get-ChildItem $storage -Recurse -Filter history_item.json -ErrorAction SilentlyContinue | + Where-Object { [DateTimeOffset]$_.LastWriteTimeUtc -ge [DateTimeOffset]::FromUnixTimeSeconds($since) } | + ForEach-Object { Copy-Item $_.FullName (Join-Path $copy ($_.Directory.Name + '.json')) } + $arguments = @("$PSScriptRoot/check_history_store.py", '--storage', $storage, '--since', $since, + '--provider-log', "$EvidenceDir/provider.jsonl", '--report', "$EvidenceDir/history-check.json") + foreach ($tag in 'A', 'B') { $arguments += @('--ide-log', "$($instances[$tag].Log)/idea.log") } + & python @arguments + if ($LASTEXITCODE -ne 0) { throw 'The shared history store failed the concurrency check; see history-check.json' } + Write-Output 'PASS: two IDEs, parent tasks with subtasks, one shared history store' + return + } $product = Get-Content "$IdeDir/product-info.json" -Raw | ConvertFrom-Json $launcher = if ($product.productCode -eq 'WS') { 'webstorm64.exe' } else { 'idea64.exe' } if ($product.productCode -eq 'WS') { @@ -252,6 +351,12 @@ idea.plugins.path=$profilePath/plugins if ($ideProcess -and -not $ideProcess.HasExited) { & taskkill.exe /PID $ideProcess.Id /T /F | Out-Null } if ($fixtureProcess -and -not $fixtureProcess.HasExited) { Stop-Process -Id $fixtureProcess.Id -Force -ErrorAction SilentlyContinue } if (Test-Path "$runDir/log") { Copy-Item "$runDir/log" "$EvidenceDir/ide-log" -Recurse -Force } + foreach ($tag in 'A', 'B') { + if (Test-Path "$runDir/ide-$tag/log") { Copy-Item "$runDir/ide-$tag/log" "$EvidenceDir/ide-log-$tag" -Recurse -Force } + } + foreach ($instance in $instances.Values) { + if ($instance.Process -and -not $instance.Process.HasExited) { & taskkill.exe /PID $instance.Process.Id /T /F | Out-Null } + } $upstreamKey = $null $env:SOLHEIM_RELAY_TOKEN = $originalRelayToken $env:SOLHEIM_KEY_FILE = $originalKeyFile diff --git a/jetbrains_plugin/scripts/ui-harness/streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/streaming_fixture.py index 3dde3b1..07d7af8 100644 --- a/jetbrains_plugin/scripts/ui-harness/streaming_fixture.py +++ b/jetbrains_plugin/scripts/ui-harness/streaming_fixture.py @@ -7,11 +7,37 @@ import argparse import http.server import json +import re import threading import time from pathlib import Path +def _text(content): + if isinstance(content, list): + return ''.join(part.get('text', '') for part in content if isinstance(part, dict)) + return content if isinstance(content, str) else '' + + +def history_role(messages): + """Classify a request in the history-race scenario. + + The first user message names the task: a parent carries HISTORY_RACE_PARENT + and a child carries HISTORY_RACE_CHILD . A parent that already holds a + new_task call in a non-system message has delegated, so it resumes. + """ + users = [m for m in messages if m.get('role') == 'user'] + first = _text(users[0].get('content')) if users else '' + child = re.search(r'HISTORY_RACE_CHILD (\w+)', first) + if child: + return 'child', child.group(1) + parent = re.search(r'HISTORY_RACE_PARENT (\w+)', first) + if not parent: + return 'plain', '' + body = json.dumps([m for m in messages if m.get('role') != 'system']) + return ('resume' if 'new_task' in body else 'delegate'), parent.group(1) + + class Fixture(http.server.BaseHTTPRequestHandler): def log_message(self, *_): pass @@ -54,7 +80,10 @@ def do_POST(self): with self.server.counter_lock: self.server.requests += 1 turn = self.server.requests - self.server.record(event='request', turn=turn, tools=tools) + role, tag = 'plain', '' + if self.server.scenario == 'history-race': + role, tag = history_role(request.get('messages', [])) + self.server.record(event='request', turn=turn, tools=tools, role=role, tag=tag) self.send_response(200) self.send_header('Content-Type', 'text/event-stream') self.send_header('Cache-Control', 'no-cache') @@ -69,26 +98,44 @@ def emit(delta, finish=None): try: emit({'role': 'assistant', 'content': ''}) - for index in range(self.server.chunks): + chunks = self.server.chunks + if role == 'delegate': + # Hold every parent until all IDEs have submitted, so the delegations + # and the child streams that follow overlap in time. + try: + self.server.barrier.wait() + except threading.BrokenBarrierError: + self.server.record(event='barrier-timeout', turn=turn, tag=tag) + return + self.server.record(event='barrier-released', turn=turn, tag=tag) + if role in ('delegate', 'resume'): + chunks = 3 + for index in range(chunks): emit({'content': f'History update {index + 1} for turn {turn}. '}) time.sleep(self.server.interval) - emit({'tool_calls': [{'index': 0, 'id': f'complete-{turn}', + if role == 'delegate': + call = ('new_task', {'mode': 'code', 'todos': None, + 'message': f'HISTORY_RACE_CHILD {tag} reply with attempt_completion.'}) + else: + call = ('attempt_completion', {'result': f'Fixture turn {turn} complete.'}) + emit({'tool_calls': [{'index': 0, 'id': f'call-{turn}', 'type': 'function', 'function': { - 'name': 'attempt_completion', - 'arguments': json.dumps({'result': f'Fixture turn {turn} complete.'})}}]}) + 'name': call[0], 'arguments': json.dumps(call[1])}}]}) emit({}, 'tool_calls') self.wfile.write(b'data: [DONE]\n\n') self.wfile.flush() - self.server.record(event='completed', turn=turn, chunks=self.server.chunks) + self.server.record(event='completed', turn=turn, chunks=chunks) except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError): self.server.record(event='disconnected', turn=turn) -def make_server(port=19487, log=None, chunks=100, interval=0.25): +def make_server(port=19487, log=None, chunks=100, interval=0.25, scenario=None, barrier=2, barrier_timeout=300): server = http.server.ThreadingHTTPServer(('127.0.0.1', port), Fixture) server.requests = 0 server.counter_lock = threading.Lock() server.chunks, server.interval = chunks, interval + server.scenario = scenario + server.barrier = threading.Barrier(barrier, timeout=barrier_timeout) def record(**fields): if log: @@ -102,5 +149,9 @@ def record(**fields): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--port', type=int, default=19487) parser.add_argument('--log', type=Path, required=True) + parser.add_argument('--scenario', choices=['history-race']) + parser.add_argument('--chunks', type=int, default=100) + parser.add_argument('--interval', type=float, default=0.25) + parser.add_argument('--barrier', type=int, default=2) args = parser.parse_args() - make_server(args.port, args.log).serve_forever() + make_server(args.port, args.log, args.chunks, args.interval, args.scenario, args.barrier).serve_forever() diff --git a/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py b/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py new file mode 100644 index 0000000..65da66b --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py @@ -0,0 +1,244 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from check_history_store import check_items, check_logs, check_provider, load_items + + +def item(task_id, text, **extra): + return {'id': task_id, 'task': text, 'size': 10, 'workspace': 'w', **extra} + + +def good_pair(tag, workspace): + return { + 'p' + tag: item('p' + tag, f'HISTORY_RACE_PARENT {tag} go', childIds=['c' + tag], + status='active', workspace=workspace), + 'c' + tag: item('c' + tag, f'HISTORY_RACE_CHILD {tag} go', parentTaskId='p' + tag, + status='completed', workspace=workspace), + } + + +# Real items from Windows CI run 43562237505 (commit 5a494cf, IDEA 2026.2.1, extension 3.96.0). +# Only the temporary run folder name in `workspace` was shortened. +REAL_ITEMS = json.loads(r'''[ + { + "id": "01a1d898-c7ab-70da-936e-6467341e0125", + "parentTaskId": "01a1d898-bf28-71a1-92d9-df474bf72871", + "number": 1, + "ts": 1794635304779, + "task": "HISTORY_RACE_CHILD A reply with attempt_completion.", + "tokensIn": 0, + "tokensOut": 0, + "totalCost": 0, + "size": 10621, + "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-A", + "mode": "code", + "apiConfigName": "focus-fixture", + "status": "completed", + "completionResultSummary": "Fixture turn 4 complete." + }, + { + "id": "01a1d898-c663-72e9-b6dd-1caba4bda93a", + "parentTaskId": "01a1d898-4435-75fd-afc2-6ca3aa4c89a4", + "number": 1, + "ts": 1794635304331, + "task": "HISTORY_RACE_CHILD B reply with attempt_completion.", + "tokensIn": 0, + "tokensOut": 0, + "totalCost": 0, + "size": 10623, + "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-B", + "mode": "code", + "apiConfigName": "focus-fixture", + "status": "completed", + "completionResultSummary": "Fixture turn 3 complete." + }, + { + "id": "01a1d898-bf28-71a1-92d9-df474bf72871", + "number": 1, + "ts": 1794635306508, + "task": "HISTORY_RACE_PARENT A delegate one subtask, then finish.", + "tokensIn": 0, + "tokensOut": 0, + "totalCost": 0, + "size": 7694, + "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-A", + "mode": "code", + "apiConfigName": "focus-fixture", + "status": "active", + "childIds": [ + "01a1d898-c7ab-70da-936e-6467341e0125" + ], + "completedByChildId": "01a1d898-c7ab-70da-936e-6467341e0125", + "completionResultSummary": "Fixture turn 4 complete." + }, + { + "id": "01a1d898-4435-75fd-afc2-6ca3aa4c89a4", + "number": 1, + "ts": 1794635306029, + "task": "HISTORY_RACE_PARENT B delegate one subtask, then finish.", + "tokensIn": 0, + "tokensOut": 0, + "totalCost": 0, + "size": 7698, + "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-B", + "mode": "code", + "apiConfigName": "focus-fixture", + "status": "active", + "childIds": [ + "01a1d898-c663-72e9-b6dd-1caba4bda93a" + ], + "completedByChildId": "01a1d898-c663-72e9-b6dd-1caba4bda93a", + "completionResultSummary": "Fixture turn 3 complete." + } +]''') + + +class CheckItemsTest(unittest.TestCase): + def test_real_extension_items_pass(self): + items = {i['id']: i for i in REAL_ITEMS} + self.assertEqual(4, len(items)) + self.assertEqual([], check_items(items, 2)) + + + def two(self): + return {**good_pair('A', 'wa'), **good_pair('B', 'wb')} + + def test_clean_store_passes(self): + self.assertEqual([], check_items(self.two(), 2)) + + def test_placeholder_title_and_zero_size_fail(self): + items = self.two() + items['pA']['task'] = 'Task #1' + items['cB']['size'] = 0 + failures = check_items(items, 2) + self.assertTrue(any('placeholder' in f for f in failures)) + self.assertTrue(any('size is 0' in f for f in failures)) + + def test_cross_linked_children_fail(self): + items = self.two() + items['pA']['childIds'] = ['cB'] + items['cA']['parentTaskId'] = 'pB' + failures = check_items(items, 2) + self.assertTrue(any('childIds' in f for f in failures)) + self.assertTrue(any('parentTaskId' in f for f in failures)) + + def test_foreign_or_missing_child_ids_fail(self): + for child_ids in (['cA', 'cB'], []): + items = self.two() + items['pA']['childIds'] = child_ids + self.assertTrue(any('childIds' in f for f in check_items(items, 2)), child_ids) + + def test_stuck_delegation_and_open_child_fail(self): + items = self.two() + items['pA'].update(status='delegated', awaitingChildId='cA') + items['cA']['status'] = 'interrupted' + failures = check_items(items, 2) + self.assertTrue(any('still waits' in f for f in failures)) + self.assertTrue(any('child status' in f for f in failures)) + + def fails(self, items, message): + self.assertIn(message, check_items(items, 2)) + + def test_each_bad_item_branch_reports_an_exact_message(self): + items = self.two() + items['pA']['task'] = 'Work #2' + self.fails(items, "pA: placeholder or empty title 'Work #2'") + items = self.two() + items['pA']['task'] = ' ' + self.fails(items, "pA: placeholder or empty title ' '") + items = self.two() + items['pA']['task'] = 'no marker' + self.fails(items, "pA: title does not carry a run marker: 'no marker'") + items = self.two() + items['pA']['delegatedToId'] = 'cA' + self.fails(items, 'pA: parent still waits on a child (status \'active\')') + items = self.two() + items['pA']['awaitingChildId'] = 'cA' + self.fails(items, 'pA: parent still waits on a child (status \'active\')') + items = self.two() + items['cA']['workspace'] = 'other' + self.fails(items, 'pA: parent and child workspaces differ') + + def test_shared_workspace_and_extra_child_fail(self): + items = {**good_pair('A', 'w'), **good_pair('B', 'w')} + self.assertTrue(any('distinct workspaces' in f for f in check_items(items, 2))) + items = self.two() + items.update(good_pair('A', 'wa')) + items['cA2'] = item('cA2', 'HISTORY_RACE_CHILD A again', parentTaskId='pA', + status='completed', workspace='wa') + self.fails(items, 'pA: expected one child for tag A, found 2') + + def test_missing_task_fails(self): + items = self.two() + del items['cB'] + self.assertTrue(any('expected 2 parents' in f for f in check_items(items, 2))) + + +class StoreAndLogTest(unittest.TestCase): + def test_index_file_and_bad_json_fail(self): + with tempfile.TemporaryDirectory() as directory: + tasks = Path(directory) / 'ext' / 'tasks' + (tasks / 'x').mkdir(parents=True) + (tasks / '_index.json').write_text('{}') + (tasks / 'x' / 'history_item.json').write_text('{broken') + items, failures = load_items(Path(directory), 0) + self.assertEqual({}, items) + self.assertEqual(2, len(failures)) + + def test_id_mismatch_and_non_object_fail(self): + with tempfile.TemporaryDirectory() as directory: + tasks = Path(directory) / 'ext' / 'tasks' + for name, text in (('x', '{"id": "y"}'), ('z', '[1]')): + (tasks / name).mkdir(parents=True) + (tasks / name / 'history_item.json').write_text(text) + items, failures = load_items(Path(directory), 0) + self.assertEqual({}, items) + self.assertEqual(["x: id field is 'y'", 'z: history_item.json is not an object'], failures) + + def test_old_tasks_are_ignored(self): + with tempfile.TemporaryDirectory() as directory: + task = Path(directory) / 'ext' / 'tasks' / 'x' + task.mkdir(parents=True) + (task / 'history_item.json').write_text('{broken') + self.assertEqual(({}, []), load_items(Path(directory), 2 ** 40)) + + def test_provider_and_log_checks(self): + with tempfile.TemporaryDirectory() as directory: + provider = Path(directory) / 'p.jsonl' + good = ['barrier-released'] * 2 + ['completed'] * 6 + provider.write_text('\n'.join(json.dumps({'event': e}) for e in good)) + self.assertEqual([], check_provider(provider, 2)) + provider.write_text(json.dumps({'event': 'barrier-timeout'})) + self.assertTrue(check_provider(provider, 2)) + def provider_failures(events): + provider.write_text('\n'.join(json.dumps({'event': e}) for e in events)) + return check_provider(provider, 2) + self.assertIn('provider logged disconnected', provider_failures(good + ['disconnected'])) + self.assertIn('barrier released 1 times, expected 2', + provider_failures(['barrier-released'] + ['completed'] * 6)) + self.assertIn('5 completed streams, expected 6', + provider_failures(['barrier-released'] * 2 + ['completed'] * 5)) + self.assertIn('7 completed streams, expected 6', provider_failures(good + ['completed'])) + self.assertIn('barrier released 3 times, expected 2', provider_failures(good + ['barrier-released'])) + self.assertEqual(['provider logged barrier-timeout'], provider_failures(good + ['barrier-timeout'])) + provider.write_text('{"event": "completed"}\n{broken') + self.assertTrue(any('not valid JSON' in f for f in check_provider(provider, 2))) + provider.write_text('[1]\n' + '\n'.join(json.dumps({'event': e}) for e in good)) + self.assertEqual(["p.jsonl: provider log line is not an object: [1]"], check_provider(provider, 2)) + missing = Path(directory) / 'missing.log' + self.assertEqual([f'IDE log does not exist: {missing}'], check_logs([missing])) + log = Path(directory) / 'idea.log' + log.write_text('ok\nInvalid status transition interrupted -> delegated\n') + self.assertEqual(1, len(check_logs([log]))) + for phrase in ('invalid status transition', 'DeltaRejectedError', 'failed to persist', 'Cannot re-delegate'): + log.write_text(f'ok\n{phrase}\n') + self.assertEqual([f'idea.log:2: {phrase}'], check_logs([log]), phrase) + stamped = '2026-01-01 12:00:00,000 [ 1] WARN - Invalid status transition x' + log.write_text(f'ok\n{stamped}\n') + self.assertEqual([f'idea.log:2: {stamped}'], check_logs([log])) + + +if __name__ == '__main__': + unittest.main() diff --git a/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py index 7989837..23428d4 100644 --- a/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py +++ b/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py @@ -6,7 +6,7 @@ from unittest.mock import patch from pathlib import Path -from streaming_fixture import make_server +from streaming_fixture import history_role, make_server class StreamingFixtureTest(unittest.TestCase): @@ -93,5 +93,99 @@ def test_health_and_model_discovery(self): self.assertEqual('focus-fixture', json.loads(response.read())['data'][0]['id']) +class HistoryRaceScenarioTest(unittest.TestCase): + def user(self, text): + return {'role': 'user', 'content': [{'type': 'text', 'text': text}]} + + def test_roles_follow_the_first_user_message(self): + self.assertEqual(('delegate', 'A'), history_role([self.user('HISTORY_RACE_PARENT A go')])) + self.assertEqual(('child', 'B'), history_role([self.user('HISTORY_RACE_CHILD B go')])) + self.assertEqual(('plain', ''), history_role([self.user('hello')])) + + def test_parent_resumes_after_new_task_but_ignores_the_system_prompt(self): + system = {'role': 'system', 'content': 'tools: new_task, attempt_completion'} + first = self.user('HISTORY_RACE_PARENT A go') + self.assertEqual(('delegate', 'A'), history_role([system, first])) + called = {'role': 'assistant', 'tool_calls': [{'function': {'name': 'new_task'}}]} + self.assertEqual(('resume', 'A'), history_role([system, first, called])) + + def test_two_parents_delegate_together_and_children_complete(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + log = Path(directory.name) / 'provider.jsonl' + server = make_server(0, log, chunks=2, interval=0.01, scenario='history-race', barrier=2) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + tools = [{'type': 'function', 'function': {'name': 'attempt_completion'}}] + results = {} + + def call(tag, text, extra=()): + extra = list(extra) + connection = http.client.HTTPConnection('127.0.0.1', server.server_port, timeout=10) + body = {'stream': True, 'tools': tools, 'messages': [self.user(text)] + extra} + connection.request('POST', '/v1/chat/completions', json.dumps(body), + {'Authorization': 'Bearer focus-fixture-local-only'}) + data = connection.getresponse().read().decode() + results[tag] = [json.loads(line[6:]) for line in data.splitlines() if line.startswith('data: {')] + connection.close() + + def events(): + return [json.loads(line)['event'] for line in log.read_text().splitlines()] if log.exists() else [] + + parents = {tag: threading.Thread(target=call, args=(tag, f'HISTORY_RACE_PARENT {tag}')) for tag in 'AB'} + # Parent A alone must stay blocked: no new_task and no barrier-released. + parents['A'].start() + parents['A'].join(0.5) + self.assertTrue(parents['A'].is_alive()) + self.assertNotIn('A', results) + self.assertNotIn('barrier-released', events()) + parents['B'].start() + for parent in parents.values(): + parent.join(15) + self.assertFalse(parent.is_alive()) + for tag in 'AB': + call_delta = results[tag][-2]['choices'][0]['delta']['tool_calls'][0]['function'] + self.assertEqual('new_task', call_delta['name']) + arguments = json.loads(call_delta['arguments']) + self.assertEqual(f'HISTORY_RACE_CHILD {tag} reply with attempt_completion.', arguments['message']) + self.assertIsNone(arguments['todos']) + call('A', 'HISTORY_RACE_CHILD A go') + done = results['A'][-2]['choices'][0]['delta']['tool_calls'][0]['function']['name'] + self.assertEqual('attempt_completion', done) + called = {'role': 'assistant', 'tool_calls': [{'function': {'name': 'new_task'}}]} + call('A', 'HISTORY_RACE_PARENT A go', [called]) + events = [json.loads(line)['event'] for line in log.read_text().splitlines()] + self.assertEqual(2, events.count('barrier-released')) + self.assertNotIn('barrier-timeout', events) + counts = [json.loads(line)['chunks'] for line in log.read_text().splitlines() + if json.loads(line)['event'] == 'completed'] + self.assertEqual([2, 3, 3, 3], sorted(counts)) + + def test_lone_parent_times_out_instead_of_passing(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + log = Path(directory.name) / 'provider.jsonl' + server = make_server(0, log, chunks=1, interval=0.01, scenario='history-race', + barrier=2, barrier_timeout=0.3) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + connection = http.client.HTTPConnection('127.0.0.1', server.server_port, timeout=10) + body = {'stream': True, 'tools': [{'type': 'function', 'function': {'name': 'attempt_completion'}}], + 'messages': [self.user('HISTORY_RACE_PARENT A go')]} + connection.request('POST', '/v1/chat/completions', json.dumps(body), + {'Authorization': 'Bearer focus-fixture-local-only'}) + connection.getresponse().read() + connection.close() + events = [json.loads(line)['event'] for line in log.read_text().splitlines()] + self.assertIn('barrier-timeout', events) + self.assertNotIn('barrier-released', events) + + if __name__ == '__main__': unittest.main()