From 5bae728b03eaaad6c9e6bb6fb5286bc29b945eec Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 14 Nov 2026 04:00:32 +0000 Subject: [PATCH 1/5] test: add two-IDE shared history race test with subtasks --- .github/workflows/windows-chat-focus.yml | 40 +++- .../scripts/ui-harness/HarnessDriver.java | 51 ++++- jetbrains_plugin/scripts/ui-harness/README.md | 19 ++ .../scripts/ui-harness/check_history_store.py | 131 ++++++++++++ .../scripts/ui-harness/run-history-race.sh | 200 ++++++++++++++++++ .../scripts/ui-harness/run-windows.ps1 | 105 ++++++++- .../scripts/ui-harness/streaming_fixture.py | 65 +++++- .../ui-harness/test_check_history_store.py | 91 ++++++++ .../ui-harness/test_streaming_fixture.py | 80 ++++++- 9 files changed, 764 insertions(+), 18 deletions(-) create mode 100644 jetbrains_plugin/scripts/ui-harness/check_history_store.py create mode 100755 jetbrains_plugin/scripts/ui-harness/run-history-race.sh create mode 100644 jetbrains_plugin/scripts/ui-harness/test_check_history_store.py diff --git a/.github/workflows/windows-chat-focus.yml b/.github/workflows/windows-chat-focus.yml index 7d3136e..7d450b1 100644 --- a/.github/workflows/windows-chat-focus.yml +++ b/.github/workflows/windows-chat-focus.yml @@ -27,6 +27,12 @@ on: permissions: contents: read +env: + # Reproduce the release under investigation with the current Kotlin changes. + # The build job packages this extension version. The test steps check that the + # plugin ZIP really contains it, because the ZIP name does not set it. + ZOO_EXTENSION_VERSION: "3.96.0" + concurrency: # Skipped label events must not replace a pending build for this PR. group: windows-chat-focus-${{ github.ref }}-${{ github.event.action == 'labeled' && github.event.label.name != 'live-solheim' && 'ignored-label' || 'validation' }} @@ -40,8 +46,6 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 90 env: - # Reproduce the release under investigation with the current Kotlin changes. - ZOO_EXTENSION_VERSION: "3.96.0" # This job packages the plugin; headful tests use the IDE's bundled JCEF. PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" steps: @@ -82,10 +86,10 @@ jobs: if-no-files-found: error windows-e2e: - name: Native typing and streaming chat (IDEA ${{ matrix.ide }}) + name: Native typing, streaming chat, and shared history (IDEA ${{ matrix.ide }}) needs: build runs-on: windows-2022 - timeout-minutes: 30 + timeout-minutes: 60 strategy: fail-fast: false matrix: @@ -104,8 +108,10 @@ jobs: - uses: actions/setup-python@v6 with: python-version: "3.12" - - name: Validate fixture HTTP behavior on Windows - run: python -m unittest discover -s jetbrains_plugin/scripts/ui-harness -p test_streaming_fixture.py -v + - name: Validate fixture HTTP behavior and the history store checker on Windows + run: | + python -m unittest discover -s jetbrains_plugin/scripts/ui-harness -p test_streaming_fixture.py -v + python -m unittest discover -s jetbrains_plugin/scripts/ui-harness -p test_check_history_store.py -v - uses: actions/download-artifact@v4 with: name: chat-focus-plugin @@ -138,7 +144,21 @@ jobs: if ($plugins.Count -ne 1) { throw 'Expected exactly one plugin distribution' } & ./jetbrains_plugin/scripts/ui-harness/run-windows.ps1 ` -IdeDir $env:FOCUS_IDE_DIR -PluginZip $plugins[0].FullName ` + -ExpectExtensionVersion $env:ZOO_EXTENSION_VERSION ` -EvidenceDir "$env:GITHUB_WORKSPACE/focus-evidence" + # Issue #6. Two IDEs share ~/.zoo-code. Each one runs a parent task that + # delegates to a subtask. The step fails if the shared history store has a + # shared index file, invalid items, wrong parent/child links, a subtask view + # that flips back, or an invalid status transition in the IDE logs. + - name: Run two IDEs with parent tasks and subtasks against one history store + if: ${{ !cancelled() }} + run: | + $plugins = @(Get-ChildItem focus-plugin -Filter *.zip) + if ($plugins.Count -ne 1) { throw 'Expected exactly one plugin distribution' } + & ./jetbrains_plugin/scripts/ui-harness/run-windows.ps1 -HistoryRace ` + -IdeDir $env:FOCUS_IDE_DIR -PluginZip $plugins[0].FullName ` + -ExpectExtensionVersion $env:ZOO_EXTENSION_VERSION ` + -EvidenceDir "$env:GITHUB_WORKSPACE/history-evidence" -DeadlineSeconds 900 - name: Upload focus traces, screenshots, and run identity if: always() uses: actions/upload-artifact@v4 @@ -146,6 +166,13 @@ jobs: name: windows-chat-focus-${{ matrix.ide }}-${{ github.sha }} path: focus-evidence/ if-no-files-found: error + - name: Upload shared history evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: windows-history-race-${{ matrix.ide }}-${{ github.sha }} + path: history-evidence/ + if-no-files-found: warn live-solheim: name: Live Solheim terminal and diff focus (WebStorm 2026.2.1) @@ -201,6 +228,7 @@ jobs: $env:SOLHEIM_API_KEY = $liveKey & ./jetbrains_plugin/scripts/ui-harness/run-windows.ps1 ` -IdeDir $env:FOCUS_IDE_DIR -PluginZip $plugins[0].FullName ` + -ExpectExtensionVersion $env:ZOO_EXTENSION_VERSION ` -Provider solheim -DeadlineSeconds 780 -RequireWindows11:($env:REQUIRE_WINDOWS11 -eq 'true') ` -EvidenceDir "$env:GITHUB_WORKSPACE/live-evidence/run-$iteration" } diff --git a/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java b/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java index f03eb2e..9066b5d 100644 --- a/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java +++ b/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java @@ -49,6 +49,10 @@ public final class HarnessDriver implements AppLifecycleListener { private final java.util.Set approvedCommands = new java.util.HashSet<>(); private final boolean fixtureMode = Boolean.getBoolean("zoo.harness.fixture"); private final boolean diffStress = Boolean.getBoolean("zoo.harness.diffStress"); + // Issue #6: two IDEs share one history store. This mode submits one parent task that + // delegates to a subtask, then waits for the parent to resume. It does no focus checks. + private final boolean historyRace = Boolean.getBoolean("zoo.harness.historyRace"); + private final String historyTag = System.getProperty("zoo.harness.historyTag", "A"); private final boolean windowsLive = !fixtureMode && System.getProperty("os.name", "").startsWith("Windows"); private String windowsCommand(String step) { @@ -248,7 +252,7 @@ private void mark(String step) { // Manual provider-backed battery: real React textarea, real extension tools. private JsonObject realSnapshot(Object webview, AtomicReference title) throws Exception { int id=++realSequence; - String script="(()=>{const snapshot=JSON.stringify((()=>{let e=[...document.querySelectorAll('textarea')].find(x=>x.offsetWidth&&x.offsetHeight);let r=e&&e.getBoundingClientRect();let z=window.__focusE2E||{};let m=z.messages||[];let last=m[m.length-1]||{};return {snapshot:"+id+",input:!!e,value:e?e.value:null,active:e===document.activeElement,windowFocused:document.hasFocus(),disabled:e?e.disabled:null,rect:r?{x:r.x,y:r.y,w:r.width,h:r.height}:null,messages:m.length,events:z.events||0,partial:z.partial||0,lastAsk:last.ask||null,lastTs:last.ts||null,lastText:typeof last.text==='string'?last.text.slice(0,300):null,lastSay:last.say||null,lastPartial:!!last.partial,provider:z.provider||null,body:document.body.innerText.slice(-1200)};})());console.log('ZFE2E:'+snapshot);document.title='ZFE2E:'+snapshot;})();"; + String script="(()=>{const snapshot=JSON.stringify((()=>{let e=[...document.querySelectorAll('textarea')].find(x=>x.offsetWidth&&x.offsetHeight);let r=e&&e.getBoundingClientRect();let z=window.__focusE2E||{};let m=z.messages||[];let last=m[m.length-1]||{};return {snapshot:"+id+",input:!!e,value:e?e.value:null,active:e===document.activeElement,windowFocused:document.hasFocus(),disabled:e?e.disabled:null,rect:r?{x:r.x,y:r.y,w:r.width,h:r.height}:null,messages:m.length,events:z.events||0,partial:z.partial||0,taskId:z.taskId||null,tasks:z.tasks||[],lastAsk:last.ask||null,lastTs:last.ts||null,lastText:typeof last.text==='string'?last.text.slice(0,300):null,lastSay:last.say||null,lastPartial:!!last.partial,provider:z.provider||null,body:document.body.innerText.slice(-1200)};})());console.log('ZFE2E:'+snapshot);document.title='ZFE2E:'+snapshot;})();"; runOnEdt(() -> invoke(webview, "executeJavaScript", script)); long resend=System.currentTimeMillis()+1000; long deadline=System.currentTimeMillis()+30000; @@ -318,7 +322,7 @@ private void runRealConversation(Project project) throws Exception { if (!addTitleHandler(browser, invoke(browser, "getCefBrowser"), title)) { throw new IllegalStateException("Cannot register JCEF snapshot handler; see IDE log for cause"); } - invoke(webview, "executeJavaScript", "window.__focusE2E={messages:[],events:0,partial:0};window.addEventListener('message',e=>{let d=e.data,z=window.__focusE2E;if(!d)return;z.events++;if(d.type==='state'&&d.state){if(d.state.clineMessages)z.messages=d.state.clineMessages.slice();z.provider=d.state.apiConfiguration&&d.state.apiConfiguration.apiProvider;}if(d.type==='messageUpdated'||d.type==='partialMessage'){let m=d.clineMessage||d.partialMessage;if(m){if(m.partial)z.partial++;let i=z.messages.findIndex(x=>x.ts===m.ts);if(i>=0)z.messages[i]=m;else z.messages.push(m);}}});"); + invoke(webview, "executeJavaScript", "window.__focusE2E={messages:[],events:0,partial:0};window.addEventListener('message',e=>{let d=e.data,z=window.__focusE2E;if(!d)return;z.events++;if(d.type==='state'&&d.state){if(d.state.clineMessages)z.messages=d.state.clineMessages.slice();z.taskId=d.state.currentTaskItem?d.state.currentTaskItem.id:null;(z.tasks=z.tasks||[]);if(z.taskId&&z.tasks[z.tasks.length-1]!==z.taskId)z.tasks.push(z.taskId);z.provider=d.state.apiConfiguration&&d.state.apiConfiguration.apiProvider;}if(d.type==='messageUpdated'||d.type==='partialMessage'){let m=d.clineMessage||d.partialMessage;if(m){if(m.partial)z.partial++;let i=z.messages.findIndex(x=>x.ts===m.ts);if(i>=0)z.messages[i]=m;else z.messages.push(m);}}});"); invoke(webview,"executeJavaScript","window.__focusNodeSeq=0;function e2eNode(e){if(!e)return null;if(!e.__e2eid)e.__e2eid=++window.__focusNodeSeq;return e.tagName+'@'+e.__e2eid;}for(let kind of ['keydown','input','focusin','focusout'])document.addEventListener(kind,e=>{if(kind==='keydown'&&!['x','y'].includes(e.key))return;console.info('E2EKEY:'+JSON.stringify({kind,ts:Date.now(),key:e.key||null,mod:e.ctrlKey||e.altKey||e.metaKey||false,node:e2eNode(e.target),active:e2eNode(document.activeElement),hasFocus:document.hasFocus(),length:typeof e.target.value==='string'?e.target.value.length:null}));},true);for(let kind of ['focus','blur'])window.addEventListener(kind,e=>console.info('E2EWINDOW:'+JSON.stringify({kind,ts:Date.now(),active:e2eNode(document.activeElement),hasFocus:document.hasFocus()})),true);"); realSend(webview, "{\"type\":\"webviewDidLaunch\"}"); realSend(webview, "{\"type\":\"mode\",\"text\":\"code\"}"); @@ -335,6 +339,7 @@ private void runRealConversation(Project project) throws Exception { realSend(webview, config.toString()); realSend(webview, "{\"type\":\"updateSettings\",\"updatedSettings\":{\"autoApprovalEnabled\":true,\"alwaysAllowReadOnly\":true,\"alwaysAllowWrite\":true,\"alwaysAllowExecute\":true,\"allowedCommands\":[\"printf\",\"sleep\",\"cat\",\"pwd\"],\"terminalShellIntegrationDisabled\":false,\"destructiveCommandGuardEnabled\":false,\"maxRequests\":20,\"telemetrySetting\":\"disabled\"}}"); if (!fixtureMode) realSend(webview, "{\"type\":\"updateSettings\",\"updatedSettings\":{\"alwaysAllowExecute\":false}}"); + if (historyRace) realSend(webview, "{\"type\":\"updateSettings\",\"updatedSettings\":{\"alwaysAllowSubtasks\":true}}"); realSend(webview, "{\"type\":\"didShowAnnouncement\"}"); invoke(webview, "executeJavaScript", "[...document.querySelectorAll('button')].find(b=>b.textContent.trim()==='Close')?.click();"); JsonObject snap = null; @@ -385,7 +390,7 @@ private void runRealConversation(Project project) throws Exception { Thread.sleep(300); } Toolkit.getDefaultToolkit().getSystemClipboard().setContents( - new java.awt.datatransfer.StringSelection(fixtureMode ? "Stream the fixture response and finish with attempt_completion." : windowsLive ? "Use tools in this throwaway Windows workspace. Run execute_command with exactly: " + windowsCommand("first") + ". Read src/Main.java, then use apply_diff to replace throwaway with solheim_e2e. Run execute_command with exactly: " + windowsCommand("second") + ". Do not edit .focus-ci or files outside this workspace. Use only execute_command, read_file, apply_diff, and attempt_completion for this task. Do not create or update a todo list. Briefly explain each step and finish with attempt_completion. Do not ask questions." : "Use tools in this throwaway workspace: execute_command printf SOLHEIM_FIRST; sleep 5. Read src/Main.java, then apply_diff replacing throwaway with solheim_e2e. Execute printf SOLHEIM_SECOND; sleep 5; cat src/Main.java. No files outside this workspace. Use only execute_command, read_file, apply_diff, and attempt_completion for this task. Do not create or update a todo list. Briefly explain each step and finish with attempt_completion. Do not ask questions."),null); + new java.awt.datatransfer.StringSelection(historyRace ? "HISTORY_RACE_PARENT " + historyTag + " delegate one subtask, then finish." : fixtureMode ? "Stream the fixture response and finish with attempt_completion." : windowsLive ? "Use tools in this throwaway Windows workspace. Run execute_command with exactly: " + windowsCommand("first") + ". Read src/Main.java, then use apply_diff to replace throwaway with solheim_e2e. Run execute_command with exactly: " + windowsCommand("second") + ". Do not edit .focus-ci or files outside this workspace. Use only execute_command, read_file, apply_diff, and attempt_completion for this task. Do not create or update a todo list. Briefly explain each step and finish with attempt_completion. Do not ask questions." : "Use tools in this throwaway workspace: execute_command printf SOLHEIM_FIRST; sleep 5. Read src/Main.java, then apply_diff replacing throwaway with solheim_e2e. Execute printf SOLHEIM_SECOND; sleep 5; cat src/Main.java. No files outside this workspace. Use only execute_command, read_file, apply_diff, and attempt_completion for this task. Do not create or update a todo list. Briefly explain each step and finish with attempt_completion. Do not ask questions."),null); Robot robot = new Robot(); robot.setAutoDelay(90); robot.keyPress(KeyEvent.VK_CONTROL); robot.keyPress(KeyEvent.VK_V); @@ -398,6 +403,7 @@ private void runRealConversation(Project project) throws Exception { snap = realSnapshot(webview,title); if (!snap.get("value").isJsonNull() && !snap.get("value").getAsString().isEmpty()) { record("P_afterSubmit",snap); fail("realSubmitFailed","Enter did not submit the real chat input");return; } if (snap.get("messages").getAsInt()==0) { record("P_afterSubmit",snap); fail("realTaskMissing","No task appeared after submitting prompt");return; } + if (historyRace) { waitForHistoryRace(webview, title); return; } // Establish a single focus point before tools run; subsequent sampling never refocuses. clickInBrowser(browser,snap.getAsJsonObject("rect")); record("P_beforeDraftTyping",realSnapshot(webview,title)); @@ -503,6 +509,45 @@ private void runRealConversation(Project project) throws Exception { finish(fixtureMode ? "testP-streaming-fixture" : "testP-real-solheim-conversation", firstCompleted&&completed&&commandsCovered&&typingHealthy&&focusHealthy&&firstStreamed&&followupStreamed&&historyGrew&&fixtureChanged?"PASS":"FAIL-REAL-CONVERSATION"); } + /** + * Waits until the parent task resumes and completes after its subtask. The webview shows + * one task at a time. The expected task order is parent, child, parent. A fourth change + * of task means the view flipped back during the subtask. + */ + private void waitForHistoryRace(Object webview, AtomicReference title) throws Exception { + long deadline = System.currentTimeMillis() + 420_000; + List sequence = new ArrayList<>(); + while (System.currentTimeMillis() < deadline) { + JsonObject snap = realSnapshot(webview, title); + // The page records every task id in arrival order. Polling alone can miss the + // parent, because a delegation can finish before the first poll. + List seen = new ArrayList<>(); + snap.getAsJsonArray("tasks").forEach(id -> seen.add(id.getAsString())); + if (!seen.equals(sequence)) { + sequence = seen; + record("H_taskSequence_" + sequence.size(), sequence); + } + String ask = snap.get("lastAsk").isJsonNull() ? "" : snap.get("lastAsk").getAsString(); + if (!answerCommandAsk(webview, snap, ask)) return; + if (sequence.size() > 3) { + fail("subtaskViewFlipped", "Task view changed " + sequence.size() + " times: " + sequence); + return; + } + String current = snap.get("taskId").isJsonNull() ? null : snap.get("taskId").getAsString(); + boolean done = sequence.size() == 3 && sequence.get(2).equals(sequence.get(0)) + && sequence.get(0).equals(current) && "completion_result".equals(ask) + && !snap.get("lastPartial").getAsBoolean(); + if (done) { + // Give the extension time to persist the final history items. + Thread.sleep(5_000); + finish("testH-history-race-" + historyTag, "PASS"); + return; + } + Thread.sleep(1_000); + } + fail("historyRaceTimeout", "Parent did not resume after its subtask; task order " + sequence); + } + private void clickInBrowser(Object jbCefBrowser, JsonObject rect) throws Exception { final Point[] loc = new Point[1]; runOnEdt(() -> { diff --git a/jetbrains_plugin/scripts/ui-harness/README.md b/jetbrains_plugin/scripts/ui-harness/README.md index 0134c4c..4954932 100644 --- a/jetbrains_plugin/scripts/ui-harness/README.md +++ b/jetbrains_plugin/scripts/ui-harness/README.md @@ -42,6 +42,25 @@ it has no input for an existing release ZIP. Its IDEA matrix is 2026.1.5.1 and 2026.2.1, so those jobs do not establish compatibility with IDEA 2023.3.8. Record the exact ZIP, IDE version, and evidence directory when comparing builds. +## Shared history race test (issue #6) + +`run-windows.ps1 -HistoryRace` starts two IDEs with separate profiles and the same +home directory, so both use one `~/.zoo-code/globalStorage`. Each IDE submits a +parent task. The fixture makes the parent call `new_task`, holds both parents at a +barrier so the delegations overlap, streams both child tasks, and lets each parent +resume and finish. `check_history_store.py` then checks the store: + +1. No `_index.json` was written. +2. Every `history_item.json` is valid, has a real title, and has `size` above zero. +3. Each parent lists only its own child, and each child names only its own parent. +4. Each child is `completed` and no parent still waits on a child. +5. The provider log shows both barriers released, no disconnect, and six completed streams. +6. The IDE logs hold no invalid status transition or failed persist message. + +The driver also fails if the webview changes task more than three times. The expected +order is parent, child, parent. The unit tests run without an IDE: +`python3 -m unittest test_streaming_fixture test_check_history_store`. + ## Linux live setup Store the Solheim key in a private file outside the repository (mode 0600). diff --git a/jetbrains_plugin/scripts/ui-harness/check_history_store.py b/jetbrains_plugin/scripts/ui-harness/check_history_store.py new file mode 100644 index 0000000..94f0c19 --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/check_history_store.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Check the shared Zoo Code task history after a concurrent two-IDE run. + +Issue #6: several IDE processes write one global history store. A safe store +keeps one valid history_item.json per task, with no shared index file and with +parent and child links that stay inside one IDE's task tree. +""" +import argparse +import json +import re +import sys +from pathlib import Path + +PLACEHOLDER = re.compile(r'^\s*(Task|Work) #\d+') +BAD_LOG = re.compile(r'invalid status transition|DeltaRejectedError|failed to persist|Cannot re-delegate', re.I) +MARKER = re.compile(r'HISTORY_RACE_(PARENT|CHILD) (\w+)') + + +def load_items(storage, since): + """Return ({task id: item}, failures) for tasks written at or after `since`.""" + items, failures = {}, [] + for index in storage.glob('**/tasks/_index.json'): + if index.stat().st_mtime >= since: + failures.append(f'shared index file was written: {index}') + for path in sorted(storage.glob('**/tasks/*/history_item.json')): + if path.stat().st_mtime < since: + continue + try: + item = json.loads(path.read_text(encoding='utf-8')) + except (OSError, ValueError) as error: + failures.append(f'{path.parent.name}: history_item.json is not valid JSON ({error})') + continue + if not isinstance(item, dict): + failures.append(f'{path.parent.name}: history_item.json is not an object') + continue + if item.get('id') != path.parent.name: + failures.append(f'{path.parent.name}: id field is {item.get("id")!r}') + continue + items[item['id']] = item + return items, failures + + +def check_items(items, instances): + failures = [] + tasks = {} + for task_id, item in items.items(): + text = item.get('task') or '' + if PLACEHOLDER.match(text) or not text.strip(): + failures.append(f'{task_id}: placeholder or empty title {text!r}') + continue + match = MARKER.search(text) + if not match: + failures.append(f'{task_id}: title does not carry a run marker: {text!r}') + continue + if not item.get('size'): + failures.append(f'{task_id}: size is {item.get("size")!r} after real work') + tasks[task_id] = (match.group(1), match.group(2), item) + parents = {i: t for i, t in tasks.items() if t[0] == 'PARENT'} + children = {i: t for i, t in tasks.items() if t[0] == 'CHILD'} + if len(parents) != instances or len(children) != instances: + failures.append(f'expected {instances} parents and {instances} children, ' + f'found {len(parents)} and {len(children)}') + for parent_id, (_, tag, parent) in parents.items(): + own = [c for c, t in children.items() if t[1] == tag] + if len(own) != 1: + failures.append(f'{parent_id}: expected one child for tag {tag}, found {len(own)}') + continue + child_id = own[0] + child = children[child_id][2] + if parent.get('childIds') != [child_id]: + failures.append(f'{parent_id}: childIds {parent.get("childIds")!r}, expected {[child_id]!r}') + if child.get('parentTaskId') != parent_id: + failures.append(f'{child_id}: parentTaskId {child.get("parentTaskId")!r}, expected {parent_id!r}') + if child.get('status') != 'completed': + failures.append(f'{child_id}: child status {child.get("status")!r}, expected completed') + if parent.get('status') == 'delegated' or parent.get('awaitingChildId') or parent.get('delegatedToId'): + failures.append(f'{parent_id}: parent still waits on a child (status {parent.get("status")!r})') + if parent.get('workspace') != child.get('workspace'): + failures.append(f'{parent_id}: parent and child workspaces differ') + workspaces = {t[2].get('workspace') for t in parents.values()} + if instances > 1 and len(workspaces) != instances: + failures.append(f'expected {instances} distinct workspaces, found {sorted(map(str, workspaces))}') + return failures + + +def check_provider(path, instances): + events = [json.loads(line) for line in path.read_text(encoding='utf-8').splitlines() if line.strip()] + names = [e['event'] for e in events] + failures = [] + if names.count('barrier-released') != instances: + failures.append(f'barrier released {names.count("barrier-released")} times, expected {instances}') + for bad in ('barrier-timeout', 'disconnected'): + if bad in names: + failures.append(f'provider logged {bad}') + if names.count('completed') != 3 * instances: + failures.append(f'{names.count("completed")} completed streams, expected {3 * instances}') + return failures + + +def check_logs(paths): + failures = [] + for path in paths: + for number, line in enumerate(path.read_text(encoding='utf-8', errors='replace').splitlines(), 1): + if BAD_LOG.search(line): + failures.append(f'{path.name}:{number}: {line.strip()[:200]}') + return failures + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--storage', type=Path, required=True) + parser.add_argument('--since', type=float, required=True, help='epoch seconds when the run started') + parser.add_argument('--provider-log', type=Path, required=True) + parser.add_argument('--ide-log', type=Path, action='append', default=[]) + parser.add_argument('--instances', type=int, default=2) + parser.add_argument('--report', type=Path) + args = parser.parse_args() + items, failures = load_items(args.storage, args.since) + failures += check_items(items, args.instances) + failures += check_provider(args.provider_log, args.instances) + failures += check_logs([p for p in args.ide_log if p.exists()]) + report = {'tasks': len(items), 'failures': failures} + text = json.dumps(report, indent=2) + if args.report: + args.report.write_text(text, encoding='utf-8') + print(text) + return 1 if failures else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/jetbrains_plugin/scripts/ui-harness/run-history-race.sh b/jetbrains_plugin/scripts/ui-harness/run-history-race.sh new file mode 100755 index 0000000..82bd759 --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/run-history-race.sh @@ -0,0 +1,200 @@ +#!/bin/bash +# Issue #6: run two IDEs against one shared Zoo Code history store (Linux, Xvfb). +# Each IDE runs a parent task that delegates to a subtask. check_history_store.py +# then checks the shared store. Both IDEs use a temporary user.home, so the test +# never touches the real ~/.zoo-code. +# +# Usage: IDE_DIR=/path/to/idea PLUGIN_ZIP=/path/to/ZooCode.zip bash run-history-race.sh +# Optional: HARNESS_WM_BIN (openbox), HARNESS_HEAP_MB, DEADLINE_SECONDS, EVIDENCE_DIR, +# EXPECT_EXTENSION_VERSION (fail when the Zoo Code extension in the ZIP differs; +# the ZIP name does not set the extension version) +set -uo pipefail +SOURCE_DIR="$(cd "$(dirname "$0")" && pwd)" +: "${IDE_DIR:?Set IDE_DIR to a JCEF-enabled IntelliJ installation}" +: "${PLUGIN_ZIP:?Set PLUGIN_ZIP to the built ZooCode distribution}" +IDE_DIR="$(realpath "$IDE_DIR")" +PLUGIN_ZIP="$(realpath "$PLUGIN_ZIP")" +PRODUCT_CODE="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["productCode"])' "$IDE_DIR/product-info.json")" +case "$PRODUCT_CODE" in + WS) IDE_LAUNCHER=webstorm ;; + IU|IC) IDE_LAUNCHER=idea ;; + *) echo "Unsupported harness IDE product: $PRODUCT_CODE" >&2; exit 1 ;; +esac +DEADLINE="${DEADLINE_SECONDS:-900}" +H="$(mktemp -d "${TMPDIR:-/tmp}/zoo-history-race.XXXXXXXX")" +EVIDENCE="${EVIDENCE_DIR:-$H/evidence}" +HOME_DIR="$H/home" +STORAGE="$HOME_DIR/.zoo-code/globalStorage" +echo "Disposable harness and evidence: $H" +mkdir -p "$H/driver/classes/META-INF" "$H/plugins-dist/harness-driver/lib" "$EVIDENCE" "$HOME_DIR" + +"$IDE_DIR/jbr/bin/javac" -cp "$IDE_DIR/lib/*" -d "$H/driver/classes" "$SOURCE_DIR/HarnessDriver.java" || exit 1 +cp "$SOURCE_DIR/plugin.xml" "$H/driver/classes/META-INF/plugin.xml" +# The IDE's JBR has javac but may omit the jar tool. A plugin jar is a zip. +python3 - "$H/driver/classes" "$H/plugins-dist/harness-driver/lib/driver.jar" <<'PY' || exit 1 +import pathlib, sys, zipfile +root = pathlib.Path(sys.argv[1]) +with zipfile.ZipFile(sys.argv[2], 'w', zipfile.ZIP_DEFLATED) as jar: + for path in sorted(root.rglob('*')): + if path.is_file(): + jar.write(path, path.relative_to(root).as_posix()) +PY +unzip -q "$PLUGIN_ZIP" -d "$H/plugins-dist" || exit 1 +[ -f "$H/plugins-dist/ZooCode/runtime/extension.js" ] || { echo 'Distribution has no release extension runtime' >&2; exit 1; } +EXTENSION_VERSION="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["version"])' "$H/plugins-dist/ZooCode/zoo-code/package.json")" || exit 1 +echo "$EXTENSION_VERSION" > "$EVIDENCE/extension-version.txt" +echo "Bundled Zoo Code extension version: $EXTENSION_VERSION" +if [ -n "${EXPECT_EXTENSION_VERSION:-}" ] && [ "$EXTENSION_VERSION" != "$EXPECT_EXTENSION_VERSION" ]; then + echo "Plugin ZIP bundles Zoo Code extension $EXTENSION_VERSION, expected $EXPECT_EXTENSION_VERSION." >&2 + rm -rf "$H/plugins-dist" "$H/driver" + exit 1 +fi + +PIDS=() +cleanup() { + for pid in "${PIDS[@]}"; do + pkill -TERM -P "$pid" 2>/dev/null + kill -TERM "$pid" 2>/dev/null + done + sleep 3 + for pid in "${PIDS[@]}"; do + pkill -9 -P "$pid" 2>/dev/null + kill -9 "$pid" 2>/dev/null + done + # Remove the large copies (plugins, IDE profiles, temporary home). The evidence stays, + # and it holds a copy of the history items. + rm -rf "$H/cfg" "$H/plugins-dist" "$H/driver" "$HOME_DIR" +} +trap cleanup EXIT + +# One virtual display for both IDEs. +XVFB_NUM=$((300 + RANDOM % 500)) +Xvfb ":$XVFB_NUM" -screen 0 1600x1000x24 -nolisten tcp > "$EVIDENCE/xvfb.log" 2>&1 & +PIDS+=($!) +sleep 2 +export DISPLAY=":$XVFB_NUM" +export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/xdg-$XVFB_NUM}" +mkdir -p "$XDG_RUNTIME_DIR"; chmod 700 "$XDG_RUNTIME_DIR" +if [ -n "${HARNESS_WM_BIN:-}" ]; then + "$HARNESS_WM_BIN" --sm-disable > "$EVIDENCE/window-manager.log" 2>&1 & + PIDS+=($!) +fi + +# Local provider with the history-race scenario. +python3 "$SOURCE_DIR/streaming_fixture.py" --scenario history-race --chunks 200 \ + --log "$EVIDENCE/provider.jsonl" > "$EVIDENCE/provider-stdout.log" 2> "$EVIDENCE/provider-stderr.log" & +PIDS+=($!) +for _ in $(seq 1 40); do + curl -fsS http://127.0.0.1:19487/health > /dev/null 2>&1 && break + sleep 0.25 +done +curl -fsS http://127.0.0.1:19487/health > /dev/null 2>&1 || { echo 'Local provider did not become healthy (is port 19487 busy?)' >&2; exit 1; } + +click_startup_dialogs() { + local ev="$1" + for _ in $(seq 1 30); do + [ -f "$ev/progress.log" ] && return + for wid in $(xdotool search --onlyvisible --name '(IntelliJ IDEA|WebStorm) User Agreement' 2>/dev/null); do + geom=$(xdotool getwindowgeometry --shell "$wid" 2>/dev/null) || continue + eval "$geom" + [ -f "$ev/progress.log" ] && return + if [ "${WIDTH:-0}" -ge 550 ] && [ "${WIDTH:-0}" -le 700 ] && [ "${HEIGHT:-0}" -ge 400 ] && [ "${HEIGHT:-0}" -le 520 ]; then + xdotool mousemove $((X + 40)) $((Y + HEIGHT * 85 / 100)) click 1 + sleep 1 + xdotool mousemove $((X + WIDTH * 89 / 100)) $((Y + HEIGHT * 93 / 100)) click 1 + sleep 2 + fi + done + sleep 3 + done +} + +start_ide() { + local tag="$1" + local cfg="$H/cfg/$tag" ev="$EVIDENCE/$tag" ws="$H/workspace-$tag" + mkdir -p "$cfg/config" "$cfg/system" "$cfg/log" "$ev" "$ws/src" "$H/props" + mkdir -p "$cfg/plugins" + cp -r "$H/plugins-dist/ZooCode" "$cfg/plugins/ZooCode" + cp -r "$H/plugins-dist/harness-driver" "$cfg/plugins/harness-driver" + printf 'public class Main { public static void main(String[] a) { System.out.println("throwaway"); } }\n' > "$ws/src/Main.java" + cat > "$H/props/idea-$tag.properties" <' + echo "-Dide.show.tips.on.startup.default.value=false" + echo "-Dide.browser.jcef.extra.args=--disable-dev-shm-usage" + echo "-Dzoo.harness.mode=P" + echo "-Dzoo.harness.fixture=true" + echo "-Dzoo.harness.historyRace=true" + echo "-Dzoo.harness.historyTag=$tag" + echo "-Dzoo.harness.evidence=$ev" + } > "$H/props/vm-$tag.vmoptions" + click_startup_dialogs "$ev" & + PIDS+=($!) + ( + export HOME="$HOME_DIR" + export IDEA_PROPERTIES="$H/props/idea-$tag.properties" IDEA_VM_OPTIONS="$H/props/vm-$tag.vmoptions" + export WEBIDE_PROPERTIES="$IDEA_PROPERTIES" WEBIDE_VM_OPTIONS="$IDEA_VM_OPTIONS" + cd "$IDE_DIR/bin" + exec "./$IDE_LAUNCHER" "$ws" + ) > "$ev/launcher-stdout.log" 2>&1 & + IDE_PIDS[$tag]=$! + PIDS+=($!) +} + +# Wait for a file test or a dead IDE. $1 tag, $2 description, $3 seconds, rest: test command. +wait_for() { + local tag="$1" what="$2" seconds="$3"; shift 3 + for _ in $(seq 1 "$seconds"); do + "$@" && return 0 + if ! kill -0 "${IDE_PIDS[$tag]}" 2>/dev/null; then echo "IDE $tag exited before $what" >&2; return 1; fi + sleep 1 + done + echo "IDE $tag did not reach $what within ${seconds}s" >&2 + return 1 +} +submitted() { grep -qs 'P-real-conversation-submitted' "$EVIDENCE/$1/progress.log"; } +has_status() { [ -f "$EVIDENCE/$1/status.txt" ]; } + +declare -A IDE_PIDS +SINCE=$(( $(date +%s) - 2 )) +RESULT=FAIL + +# The driver pastes and types with a real keyboard. Start B only after A has submitted. +# A then waits at the provider barrier until B submits, so both delegations start together. +start_ide A +if wait_for A 'its first submit' 300 submitted A; then + start_ide B + ok=1 + for tag in A B; do + wait_for "$tag" 'a result' "$DEADLINE" has_status "$tag" || { ok=0; break; } + status="$(cat "$EVIDENCE/$tag/status.txt")" + [ "$status" = PASS ] || { echo "IDE $tag result: $status (see $EVIDENCE/$tag/evidence.json)" >&2; ok=0; break; } + done + if [ "$ok" = 1 ]; then + mkdir -p "$EVIDENCE/history-store" + find "$STORAGE" -name history_item.json -newermt "@$SINCE" -print0 2>/dev/null | while IFS= read -r -d '' f; do + cp "$f" "$EVIDENCE/history-store/$(basename "$(dirname "$f")").json" + done + if python3 "$SOURCE_DIR/check_history_store.py" --storage "$STORAGE" --since "$SINCE" \ + --provider-log "$EVIDENCE/provider.jsonl" --report "$EVIDENCE/history-check.json" \ + --ide-log "$H/cfg/A/log/idea.log" --ide-log "$H/cfg/B/log/idea.log"; then + RESULT=PASS + fi + fi +fi + +for tag in A B; do cp "$H/cfg/$tag/log/idea.log" "$EVIDENCE/idea-$tag.log" 2>/dev/null || true; done +echo "HISTORY-RACE result: $RESULT" +echo "Evidence: $EVIDENCE" +[ "$RESULT" = PASS ] diff --git a/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 b/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 index f7c4cf8..e50cb92 100644 --- a/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 +++ b/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 @@ -6,7 +6,12 @@ param( [Parameter(Mandatory = $true)][string]$EvidenceDir, [ValidateRange(60, 1800)][int]$DeadlineSeconds = 600, [ValidateSet('fixture', 'solheim')][string]$Provider = 'fixture', - [switch]$RequireWindows11 + [switch]$RequireWindows11, + # Fail when the Zoo Code extension inside the plugin ZIP has a different version. + # The ZIP name does not set that version. + [string]$ExpectExtensionVersion, + # Issue #6: run two IDEs against one history store, each with a parent task and a subtask. + [switch]$HistoryRace ) $ErrorActionPreference = 'Stop' $IdeDir = (Resolve-Path $IdeDir).Path @@ -17,8 +22,10 @@ if (Test-Path (Join-Path $EvidenceDir 'status.txt')) { throw 'Use a fresh eviden $runDir = Join-Path ([IO.Path]::GetTempPath()) ('zoo-focus-' + [guid]::NewGuid().ToString('N')) New-Item -ItemType Directory $runDir | Out-Null $ideProcess = $null +$instances = @{} $fixtureProcess = $null $live = $Provider -eq 'solheim' +if ($HistoryRace -and $live) { throw 'The history race test uses only the local fixture provider.' } $originalRelayToken = $env:SOLHEIM_RELAY_TOKEN $originalKeyFile = $env:SOLHEIM_KEY_FILE $originalProxyLog = $env:SOLHEIM_PROXY_LOG @@ -92,6 +99,12 @@ try { if (-not (Test-Path "$plugins/ZooCode/runtime/extension.js")) { throw 'Distribution must contain the release extension-host runtime; build with debugMode=release.' } + $extensionVersion = (Get-Content "$plugins/ZooCode/zoo-code/package.json" -Raw | ConvertFrom-Json).version + Set-Content (Join-Path $EvidenceDir 'extension-version.txt') $extensionVersion + Write-Output "Bundled Zoo Code extension version: $extensionVersion" + if ($ExpectExtensionVersion -and $extensionVersion -ne $ExpectExtensionVersion) { + throw "Plugin ZIP bundles Zoo Code extension $extensionVersion, expected $ExpectExtensionVersion." + } $project = Join-Path $runDir 'workspace' New-Item -ItemType Directory -Force "$project/src" | Out-Null 'public class Main { public static void main(String[] args) { System.out.println("throwaway"); } }' | Set-Content "$project/src/Main.java" @@ -142,6 +155,7 @@ idea.plugins.path=$profilePath/plugins # Keep the provider alive without creating a foreground test obstacle. $providerScript = "$PSScriptRoot/streaming_fixture.py" $providerArguments = @("`"$providerScript`"", '--log', "`"$EvidenceDir/provider.jsonl`"") + if ($HistoryRace) { $providerArguments += @('--scenario', 'history-race', '--chunks', '200') } $readyUri = 'http://127.0.0.1:19487/health' $readyHeaders = @{} if ($live) { @@ -177,6 +191,89 @@ idea.plugins.path=$profilePath/plugins $env:SOLHEIM_KEY_FILE = $null $env:SOLHEIM_PROXY_LOG = $null } + if ($HistoryRace) { + # Both IDEs keep their own config, system, and log paths. They share the plugins + # directory, the home directory, and so ~/.zoo-code/globalStorage, as in issue #6. + $storage = Join-Path $env:USERPROFILE '.zoo-code/globalStorage' + $since = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() - 2 + $product = Get-Content "$IdeDir/product-info.json" -Raw | ConvertFrom-Json + $launcher = if ($product.productCode -eq 'WS') { 'webstorm64.exe' } else { 'idea64.exe' } + function Start-RaceIde([string]$tag) { + $profileDir = Join-Path $runDir "ide-$tag" + $workspace = Join-Path $runDir "workspace-$tag" + New-Item -ItemType Directory -Force "$workspace/src" | Out-Null + Copy-Item "$project/src/Main.java" "$workspace/src/Main.java" + $evidence = Join-Path $EvidenceDir $tag + New-Item -ItemType Directory -Force $evidence | Out-Null + $profilePath = $profileDir.Replace('\', '/') + $pluginsPath = $plugins.Replace('\', '/') + $propertiesFile = Join-Path $runDir "idea-$tag.properties" + @" +idea.config.path=$profilePath/config +idea.system.path=$profilePath/system +idea.log.path=$profilePath/log +idea.plugins.path=$pluginsPath +"@ | Set-Content $propertiesFile + $optionsFile = Join-Path $runDir "idea-$tag.vmoptions" + @" +-Xmx1024m +-XX:ActiveProcessorCount=2 +-Didea.trust.all.projects=true +-Didea.initially.ask.config=false +-Djb.consents.confirmation.enabled=false +-Djb.privacy.policy.text= +-Dide.show.tips.on.startup.default.value=false +-Dzoo.harness.mode=P +-Dzoo.harness.fixture=true +-Dzoo.harness.historyRace=true +-Dzoo.harness.historyTag=$tag +-Dzoo.harness.evidence=$($evidence.Replace('\', '/')) +"@ | Set-Content $optionsFile + $env:IDEA_PROPERTIES = $propertiesFile + $env:IDEA_VM_OPTIONS = $optionsFile + $env:IDEA64_VM_OPTIONS = $optionsFile + $env:WEBIDE_PROPERTIES = $propertiesFile + $env:WEBIDE_VM_OPTIONS = $optionsFile + $process = Start-Process -FilePath "$IdeDir/bin/$launcher" -ArgumentList "`"$workspace`"" -PassThru ` + -RedirectStandardOutput "$evidence/launcher-stdout.log" -RedirectStandardError "$evidence/launcher-stderr.log" + $instances[$tag] = [pscustomobject]@{ Process = $process; Evidence = $evidence; Log = "$profileDir/log" } + } + function Wait-RaceIde([string]$tag, [scriptblock]$until, [string]$what, [int]$seconds) { + $instance = $instances[$tag] + $limit = [DateTime]::UtcNow.AddSeconds($seconds) + while ([DateTime]::UtcNow -lt $limit) { + if (& $until $instance) { return } + if ($instance.Process.HasExited) { throw "IDE $tag exited before ${what}: $($instance.Process.ExitCode)" } + Start-Sleep -Seconds 1 + } + throw "IDE $tag did not reach $what within $seconds seconds" + } + # The driver types and pastes with a real keyboard. Two IDEs doing that together would + # steal each other's input. Start B only after A has submitted. A then waits at the + # provider barrier until B submits, so both delegations start together. + Start-RaceIde 'A' + Wait-RaceIde 'A' { param($i) (Test-Path "$($i.Evidence)/progress.log") -and + (Select-String -Path "$($i.Evidence)/progress.log" -Pattern 'P-real-conversation-submitted' -Quiet) } 'its first submit' 300 + Start-RaceIde 'B' + foreach ($tag in 'A', 'B') { + Wait-RaceIde $tag { param($i) Test-Path "$($i.Evidence)/status.txt" } 'a result' $DeadlineSeconds + $status = (Get-Content "$($instances[$tag].Evidence)/status.txt" -Raw).Trim() + if ($status -ne 'PASS') { throw "History race IDE ${tag}: $status. See $tag/evidence.json and $tag/progress.log" } + } + # Keep a copy of the store that this run wrote, for the evidence archive. + $copy = Join-Path $EvidenceDir 'history-store' + New-Item -ItemType Directory -Force $copy | Out-Null + Get-ChildItem $storage -Recurse -Filter history_item.json -ErrorAction SilentlyContinue | + Where-Object { [DateTimeOffset]$_.LastWriteTimeUtc -ge [DateTimeOffset]::FromUnixTimeSeconds($since) } | + ForEach-Object { Copy-Item $_.FullName (Join-Path $copy ($_.Directory.Name + '.json')) } + $arguments = @("$PSScriptRoot/check_history_store.py", '--storage', $storage, '--since', $since, + '--provider-log', "$EvidenceDir/provider.jsonl", '--report', "$EvidenceDir/history-check.json") + foreach ($tag in 'A', 'B') { $arguments += @('--ide-log', "$($instances[$tag].Log)/idea.log") } + & python @arguments + if ($LASTEXITCODE -ne 0) { throw 'The shared history store failed the concurrency check; see history-check.json' } + Write-Output 'PASS: two IDEs, parent tasks with subtasks, one shared history store' + return + } $product = Get-Content "$IdeDir/product-info.json" -Raw | ConvertFrom-Json $launcher = if ($product.productCode -eq 'WS') { 'webstorm64.exe' } else { 'idea64.exe' } if ($product.productCode -eq 'WS') { @@ -252,6 +349,12 @@ idea.plugins.path=$profilePath/plugins if ($ideProcess -and -not $ideProcess.HasExited) { & taskkill.exe /PID $ideProcess.Id /T /F | Out-Null } if ($fixtureProcess -and -not $fixtureProcess.HasExited) { Stop-Process -Id $fixtureProcess.Id -Force -ErrorAction SilentlyContinue } if (Test-Path "$runDir/log") { Copy-Item "$runDir/log" "$EvidenceDir/ide-log" -Recurse -Force } + foreach ($tag in 'A', 'B') { + if (Test-Path "$runDir/ide-$tag/log") { Copy-Item "$runDir/ide-$tag/log" "$EvidenceDir/ide-log-$tag" -Recurse -Force } + } + foreach ($instance in $instances.Values) { + if ($instance.Process -and -not $instance.Process.HasExited) { & taskkill.exe /PID $instance.Process.Id /T /F | Out-Null } + } $upstreamKey = $null $env:SOLHEIM_RELAY_TOKEN = $originalRelayToken $env:SOLHEIM_KEY_FILE = $originalKeyFile diff --git a/jetbrains_plugin/scripts/ui-harness/streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/streaming_fixture.py index 3dde3b1..77fb92d 100644 --- a/jetbrains_plugin/scripts/ui-harness/streaming_fixture.py +++ b/jetbrains_plugin/scripts/ui-harness/streaming_fixture.py @@ -7,11 +7,37 @@ import argparse import http.server import json +import re import threading import time from pathlib import Path +def _text(content): + if isinstance(content, list): + return ''.join(part.get('text', '') for part in content if isinstance(part, dict)) + return content if isinstance(content, str) else '' + + +def history_role(messages): + """Classify a request in the history-race scenario. + + The first user message names the task: a parent carries HISTORY_RACE_PARENT + and a child carries HISTORY_RACE_CHILD . A parent that already holds a + new_task call in a non-system message has delegated, so it resumes. + """ + users = [m for m in messages if m.get('role') == 'user'] + first = _text(users[0].get('content')) if users else '' + child = re.search(r'HISTORY_RACE_CHILD (\w+)', first) + if child: + return 'child', child.group(1) + parent = re.search(r'HISTORY_RACE_PARENT (\w+)', first) + if not parent: + return 'plain', '' + body = json.dumps([m for m in messages if m.get('role') != 'system']) + return ('resume' if 'new_task' in body else 'delegate'), parent.group(1) + + class Fixture(http.server.BaseHTTPRequestHandler): def log_message(self, *_): pass @@ -54,7 +80,10 @@ def do_POST(self): with self.server.counter_lock: self.server.requests += 1 turn = self.server.requests - self.server.record(event='request', turn=turn, tools=tools) + role, tag = 'plain', '' + if self.server.scenario == 'history-race': + role, tag = history_role(request.get('messages', [])) + self.server.record(event='request', turn=turn, tools=tools, role=role, tag=tag) self.send_response(200) self.send_header('Content-Type', 'text/event-stream') self.send_header('Cache-Control', 'no-cache') @@ -69,13 +98,29 @@ def emit(delta, finish=None): try: emit({'role': 'assistant', 'content': ''}) - for index in range(self.server.chunks): + chunks = self.server.chunks + if role == 'delegate': + # Hold every parent until all IDEs have submitted, so the delegations + # and the child streams that follow overlap in time. + try: + self.server.barrier.wait() + except threading.BrokenBarrierError: + self.server.record(event='barrier-timeout', turn=turn, tag=tag) + return + self.server.record(event='barrier-released', turn=turn, tag=tag) + if role in ('delegate', 'resume'): + chunks = 3 + for index in range(chunks): emit({'content': f'History update {index + 1} for turn {turn}. '}) time.sleep(self.server.interval) - emit({'tool_calls': [{'index': 0, 'id': f'complete-{turn}', + if role == 'delegate': + call = ('new_task', {'mode': 'code', 'todos': None, + 'message': f'HISTORY_RACE_CHILD {tag} reply with attempt_completion.'}) + else: + call = ('attempt_completion', {'result': f'Fixture turn {turn} complete.'}) + emit({'tool_calls': [{'index': 0, 'id': f'call-{turn}', 'type': 'function', 'function': { - 'name': 'attempt_completion', - 'arguments': json.dumps({'result': f'Fixture turn {turn} complete.'})}}]}) + 'name': call[0], 'arguments': json.dumps(call[1])}}]}) emit({}, 'tool_calls') self.wfile.write(b'data: [DONE]\n\n') self.wfile.flush() @@ -84,11 +129,13 @@ def emit(delta, finish=None): self.server.record(event='disconnected', turn=turn) -def make_server(port=19487, log=None, chunks=100, interval=0.25): +def make_server(port=19487, log=None, chunks=100, interval=0.25, scenario=None, barrier=2, barrier_timeout=300): server = http.server.ThreadingHTTPServer(('127.0.0.1', port), Fixture) server.requests = 0 server.counter_lock = threading.Lock() server.chunks, server.interval = chunks, interval + server.scenario = scenario + server.barrier = threading.Barrier(barrier, timeout=barrier_timeout) def record(**fields): if log: @@ -102,5 +149,9 @@ def record(**fields): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--port', type=int, default=19487) parser.add_argument('--log', type=Path, required=True) + parser.add_argument('--scenario', choices=['history-race']) + parser.add_argument('--chunks', type=int, default=100) + parser.add_argument('--interval', type=float, default=0.25) + parser.add_argument('--barrier', type=int, default=2) args = parser.parse_args() - make_server(args.port, args.log).serve_forever() + make_server(args.port, args.log, args.chunks, args.interval, args.scenario, args.barrier).serve_forever() diff --git a/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py b/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py new file mode 100644 index 0000000..9dc8a7e --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py @@ -0,0 +1,91 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from check_history_store import check_items, check_logs, check_provider, load_items + + +def item(task_id, text, **extra): + return {'id': task_id, 'task': text, 'size': 10, 'workspace': 'w', **extra} + + +def good_pair(tag, workspace): + return { + 'p' + tag: item('p' + tag, f'HISTORY_RACE_PARENT {tag} go', childIds=['c' + tag], + status='active', workspace=workspace), + 'c' + tag: item('c' + tag, f'HISTORY_RACE_CHILD {tag} go', parentTaskId='p' + tag, + status='completed', workspace=workspace), + } + + +class CheckItemsTest(unittest.TestCase): + def two(self): + return {**good_pair('A', 'wa'), **good_pair('B', 'wb')} + + def test_clean_store_passes(self): + self.assertEqual([], check_items(self.two(), 2)) + + def test_placeholder_title_and_zero_size_fail(self): + items = self.two() + items['pA']['task'] = 'Task #1' + items['cB']['size'] = 0 + failures = check_items(items, 2) + self.assertTrue(any('placeholder' in f for f in failures)) + self.assertTrue(any('size is 0' in f for f in failures)) + + def test_cross_linked_children_fail(self): + items = self.two() + items['pA']['childIds'] = ['cB'] + items['cA']['parentTaskId'] = 'pB' + failures = check_items(items, 2) + self.assertTrue(any('childIds' in f for f in failures)) + self.assertTrue(any('parentTaskId' in f for f in failures)) + + def test_stuck_delegation_and_open_child_fail(self): + items = self.two() + items['pA'].update(status='delegated', awaitingChildId='cA') + items['cA']['status'] = 'interrupted' + failures = check_items(items, 2) + self.assertTrue(any('still waits' in f for f in failures)) + self.assertTrue(any('child status' in f for f in failures)) + + def test_missing_task_fails(self): + items = self.two() + del items['cB'] + self.assertTrue(any('expected 2 parents' in f for f in check_items(items, 2))) + + +class StoreAndLogTest(unittest.TestCase): + def test_index_file_and_bad_json_fail(self): + with tempfile.TemporaryDirectory() as directory: + tasks = Path(directory) / 'ext' / 'tasks' + (tasks / 'x').mkdir(parents=True) + (tasks / '_index.json').write_text('{}') + (tasks / 'x' / 'history_item.json').write_text('{broken') + items, failures = load_items(Path(directory), 0) + self.assertEqual({}, items) + self.assertEqual(2, len(failures)) + + def test_old_tasks_are_ignored(self): + with tempfile.TemporaryDirectory() as directory: + task = Path(directory) / 'ext' / 'tasks' / 'x' + task.mkdir(parents=True) + (task / 'history_item.json').write_text('{broken') + self.assertEqual(({}, []), load_items(Path(directory), 2 ** 40)) + + def test_provider_and_log_checks(self): + with tempfile.TemporaryDirectory() as directory: + provider = Path(directory) / 'p.jsonl' + events = ['barrier-released'] * 2 + ['completed'] * 6 + provider.write_text('\n'.join(json.dumps({'event': e}) for e in events)) + self.assertEqual([], check_provider(provider, 2)) + provider.write_text(json.dumps({'event': 'barrier-timeout'})) + self.assertTrue(check_provider(provider, 2)) + log = Path(directory) / 'idea.log' + log.write_text('ok\nInvalid status transition interrupted -> delegated\n') + self.assertEqual(1, len(check_logs([log]))) + + +if __name__ == '__main__': + unittest.main() diff --git a/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py index 7989837..edf0695 100644 --- a/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py +++ b/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py @@ -6,7 +6,7 @@ from unittest.mock import patch from pathlib import Path -from streaming_fixture import make_server +from streaming_fixture import history_role, make_server class StreamingFixtureTest(unittest.TestCase): @@ -93,5 +93,83 @@ def test_health_and_model_discovery(self): self.assertEqual('focus-fixture', json.loads(response.read())['data'][0]['id']) +class HistoryRaceScenarioTest(unittest.TestCase): + def user(self, text): + return {'role': 'user', 'content': [{'type': 'text', 'text': text}]} + + def test_roles_follow_the_first_user_message(self): + self.assertEqual(('delegate', 'A'), history_role([self.user('HISTORY_RACE_PARENT A go')])) + self.assertEqual(('child', 'B'), history_role([self.user('HISTORY_RACE_CHILD B go')])) + self.assertEqual(('plain', ''), history_role([self.user('hello')])) + + def test_parent_resumes_after_new_task_but_ignores_the_system_prompt(self): + system = {'role': 'system', 'content': 'tools: new_task, attempt_completion'} + first = self.user('HISTORY_RACE_PARENT A go') + self.assertEqual(('delegate', 'A'), history_role([system, first])) + called = {'role': 'assistant', 'tool_calls': [{'function': {'name': 'new_task'}}]} + self.assertEqual(('resume', 'A'), history_role([system, first, called])) + + def test_two_parents_delegate_together_and_children_complete(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + log = Path(directory.name) / 'provider.jsonl' + server = make_server(0, log, chunks=2, interval=0.01, scenario='history-race', barrier=2) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + tools = [{'type': 'function', 'function': {'name': 'attempt_completion'}}] + results = {} + + def call(tag, text): + connection = http.client.HTTPConnection('127.0.0.1', server.server_port, timeout=10) + body = {'stream': True, 'tools': tools, 'messages': [self.user(text)]} + connection.request('POST', '/v1/chat/completions', json.dumps(body), + {'Authorization': 'Bearer focus-fixture-local-only'}) + data = connection.getresponse().read().decode() + results[tag] = [json.loads(line[6:]) for line in data.splitlines() if line.startswith('data: {')] + connection.close() + + parents = [threading.Thread(target=call, args=(tag, f'HISTORY_RACE_PARENT {tag}')) for tag in 'AB'] + for parent in parents: + parent.start() + for parent in parents: + parent.join(15) + for tag in 'AB': + call_delta = results[tag][-2]['choices'][0]['delta']['tool_calls'][0]['function'] + self.assertEqual('new_task', call_delta['name']) + arguments = json.loads(call_delta['arguments']) + self.assertEqual(f'HISTORY_RACE_CHILD {tag} reply with attempt_completion.', arguments['message']) + self.assertIsNone(arguments['todos']) + call('A', 'HISTORY_RACE_CHILD A go') + done = results['A'][-2]['choices'][0]['delta']['tool_calls'][0]['function']['name'] + self.assertEqual('attempt_completion', done) + events = [json.loads(line)['event'] for line in log.read_text().splitlines()] + self.assertEqual(2, events.count('barrier-released')) + self.assertNotIn('barrier-timeout', events) + + def test_lone_parent_times_out_instead_of_passing(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + log = Path(directory.name) / 'provider.jsonl' + server = make_server(0, log, chunks=1, interval=0.01, scenario='history-race', + barrier=2, barrier_timeout=0.3) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + connection = http.client.HTTPConnection('127.0.0.1', server.server_port, timeout=10) + body = {'stream': True, 'tools': [{'type': 'function', 'function': {'name': 'attempt_completion'}}], + 'messages': [self.user('HISTORY_RACE_PARENT A go')]} + connection.request('POST', '/v1/chat/completions', json.dumps(body), + {'Authorization': 'Bearer focus-fixture-local-only'}) + connection.getresponse().read() + connection.close() + events = [json.loads(line)['event'] for line in log.read_text().splitlines()] + self.assertIn('barrier-timeout', events) + + if __name__ == '__main__': unittest.main() From 5a494cfb43d12521c5e3b5d63e16ea3498c09505 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 14 Nov 2026 05:34:47 +0000 Subject: [PATCH 2/5] fix: start history race tasks without keyboard input --- .../scripts/ui-harness/HarnessDriver.java | 18 ++++++++- jetbrains_plugin/scripts/ui-harness/README.md | 5 ++- .../scripts/ui-harness/run-history-race.sh | 38 +++++++++---------- .../scripts/ui-harness/run-windows.ps1 | 8 ++-- 4 files changed, 40 insertions(+), 29 deletions(-) diff --git a/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java b/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java index 9066b5d..fe346fb 100644 --- a/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java +++ b/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java @@ -356,6 +356,21 @@ private void runRealConversation(Project project) throws Exception { if (!Boolean.TRUE.equals(ready)) throw new IllegalStateException("Chat remained hidden during indexing"); snap = realSnapshot(webview,title); screenshot("p-configured"); + if (historyRace) { + // Start the task with a webview message, not the keyboard. Two IDEs run together, + // and a background IDE window does not get native focus on Windows. + realSend(webview, "{\"type\":\"newTask\",\"text\":\"HISTORY_RACE_PARENT " + historyTag + + " delegate one subtask, then finish.\"}"); + boolean started = false; + for (int i = 0; i < 60 && !started; i++) { + Thread.sleep(1000); + started = realSnapshot(webview, title).get("messages").getAsInt() > 0; + } + if (!started) { fail("realTaskMissing", "No task appeared after the newTask message"); return; } + mark("P-real-conversation-submitted"); + waitForHistoryRace(webview, title); + return; + } if (diffStress) checkDeliberateEditorFocus(project); // Establish native focus before typing anything. A freshly mounted OSR // surface may paint after its DOM is ready. Setup clicks are bounded; @@ -390,7 +405,7 @@ private void runRealConversation(Project project) throws Exception { Thread.sleep(300); } Toolkit.getDefaultToolkit().getSystemClipboard().setContents( - new java.awt.datatransfer.StringSelection(historyRace ? "HISTORY_RACE_PARENT " + historyTag + " delegate one subtask, then finish." : fixtureMode ? "Stream the fixture response and finish with attempt_completion." : windowsLive ? "Use tools in this throwaway Windows workspace. Run execute_command with exactly: " + windowsCommand("first") + ". Read src/Main.java, then use apply_diff to replace throwaway with solheim_e2e. Run execute_command with exactly: " + windowsCommand("second") + ". Do not edit .focus-ci or files outside this workspace. Use only execute_command, read_file, apply_diff, and attempt_completion for this task. Do not create or update a todo list. Briefly explain each step and finish with attempt_completion. Do not ask questions." : "Use tools in this throwaway workspace: execute_command printf SOLHEIM_FIRST; sleep 5. Read src/Main.java, then apply_diff replacing throwaway with solheim_e2e. Execute printf SOLHEIM_SECOND; sleep 5; cat src/Main.java. No files outside this workspace. Use only execute_command, read_file, apply_diff, and attempt_completion for this task. Do not create or update a todo list. Briefly explain each step and finish with attempt_completion. Do not ask questions."),null); + new java.awt.datatransfer.StringSelection(fixtureMode ? "Stream the fixture response and finish with attempt_completion." : windowsLive ? "Use tools in this throwaway Windows workspace. Run execute_command with exactly: " + windowsCommand("first") + ". Read src/Main.java, then use apply_diff to replace throwaway with solheim_e2e. Run execute_command with exactly: " + windowsCommand("second") + ". Do not edit .focus-ci or files outside this workspace. Use only execute_command, read_file, apply_diff, and attempt_completion for this task. Do not create or update a todo list. Briefly explain each step and finish with attempt_completion. Do not ask questions." : "Use tools in this throwaway workspace: execute_command printf SOLHEIM_FIRST; sleep 5. Read src/Main.java, then apply_diff replacing throwaway with solheim_e2e. Execute printf SOLHEIM_SECOND; sleep 5; cat src/Main.java. No files outside this workspace. Use only execute_command, read_file, apply_diff, and attempt_completion for this task. Do not create or update a todo list. Briefly explain each step and finish with attempt_completion. Do not ask questions."),null); Robot robot = new Robot(); robot.setAutoDelay(90); robot.keyPress(KeyEvent.VK_CONTROL); robot.keyPress(KeyEvent.VK_V); @@ -403,7 +418,6 @@ private void runRealConversation(Project project) throws Exception { snap = realSnapshot(webview,title); if (!snap.get("value").isJsonNull() && !snap.get("value").getAsString().isEmpty()) { record("P_afterSubmit",snap); fail("realSubmitFailed","Enter did not submit the real chat input");return; } if (snap.get("messages").getAsInt()==0) { record("P_afterSubmit",snap); fail("realTaskMissing","No task appeared after submitting prompt");return; } - if (historyRace) { waitForHistoryRace(webview, title); return; } // Establish a single focus point before tools run; subsequent sampling never refocuses. clickInBrowser(browser,snap.getAsJsonObject("rect")); record("P_beforeDraftTyping",realSnapshot(webview,title)); diff --git a/jetbrains_plugin/scripts/ui-harness/README.md b/jetbrains_plugin/scripts/ui-harness/README.md index 4954932..17b2300 100644 --- a/jetbrains_plugin/scripts/ui-harness/README.md +++ b/jetbrains_plugin/scripts/ui-harness/README.md @@ -45,8 +45,9 @@ Record the exact ZIP, IDE version, and evidence directory when comparing builds. ## Shared history race test (issue #6) `run-windows.ps1 -HistoryRace` starts two IDEs with separate profiles and the same -home directory, so both use one `~/.zoo-code/globalStorage`. Each IDE submits a -parent task. The fixture makes the parent call `new_task`, holds both parents at a +home directory, so both use one `~/.zoo-code/globalStorage`. Both IDEs start together. +Each IDE starts a parent task with a webview `newTask` message. The test uses no keyboard, +because a background IDE window gets no native focus on Windows. The fixture makes the parent call `new_task`, holds both parents at a barrier so the delegations overlap, streams both child tasks, and lets each parent resume and finish. `check_history_store.py` then checks the store: diff --git a/jetbrains_plugin/scripts/ui-harness/run-history-race.sh b/jetbrains_plugin/scripts/ui-harness/run-history-race.sh index 82bd759..1c69df7 100755 --- a/jetbrains_plugin/scripts/ui-harness/run-history-race.sh +++ b/jetbrains_plugin/scripts/ui-harness/run-history-race.sh @@ -163,34 +163,32 @@ wait_for() { echo "IDE $tag did not reach $what within ${seconds}s" >&2 return 1 } -submitted() { grep -qs 'P-real-conversation-submitted' "$EVIDENCE/$1/progress.log"; } has_status() { [ -f "$EVIDENCE/$1/status.txt" ]; } declare -A IDE_PIDS SINCE=$(( $(date +%s) - 2 )) RESULT=FAIL -# The driver pastes and types with a real keyboard. Start B only after A has submitted. -# A then waits at the provider barrier until B submits, so both delegations start together. +# The driver starts each task with a webview message and uses no keyboard, so both IDEs +# can start together. A waits at the provider barrier until B submits, so both delegations +# start together. start_ide A -if wait_for A 'its first submit' 300 submitted A; then - start_ide B - ok=1 - for tag in A B; do - wait_for "$tag" 'a result' "$DEADLINE" has_status "$tag" || { ok=0; break; } - status="$(cat "$EVIDENCE/$tag/status.txt")" - [ "$status" = PASS ] || { echo "IDE $tag result: $status (see $EVIDENCE/$tag/evidence.json)" >&2; ok=0; break; } +start_ide B +ok=1 +for tag in A B; do + wait_for "$tag" 'a result' "$DEADLINE" has_status "$tag" || { ok=0; break; } + status="$(cat "$EVIDENCE/$tag/status.txt")" + [ "$status" = PASS ] || { echo "IDE $tag result: $status (see $EVIDENCE/$tag/evidence.json)" >&2; ok=0; break; } +done +if [ "$ok" = 1 ]; then + mkdir -p "$EVIDENCE/history-store" + find "$STORAGE" -name history_item.json -newermt "@$SINCE" -print0 2>/dev/null | while IFS= read -r -d '' f; do + cp "$f" "$EVIDENCE/history-store/$(basename "$(dirname "$f")").json" done - if [ "$ok" = 1 ]; then - mkdir -p "$EVIDENCE/history-store" - find "$STORAGE" -name history_item.json -newermt "@$SINCE" -print0 2>/dev/null | while IFS= read -r -d '' f; do - cp "$f" "$EVIDENCE/history-store/$(basename "$(dirname "$f")").json" - done - if python3 "$SOURCE_DIR/check_history_store.py" --storage "$STORAGE" --since "$SINCE" \ - --provider-log "$EVIDENCE/provider.jsonl" --report "$EVIDENCE/history-check.json" \ - --ide-log "$H/cfg/A/log/idea.log" --ide-log "$H/cfg/B/log/idea.log"; then - RESULT=PASS - fi + if python3 "$SOURCE_DIR/check_history_store.py" --storage "$STORAGE" --since "$SINCE" \ + --provider-log "$EVIDENCE/provider.jsonl" --report "$EVIDENCE/history-check.json" \ + --ide-log "$H/cfg/A/log/idea.log" --ide-log "$H/cfg/B/log/idea.log"; then + RESULT=PASS fi fi diff --git a/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 b/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 index e50cb92..62fa97e 100644 --- a/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 +++ b/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 @@ -248,12 +248,10 @@ idea.plugins.path=$pluginsPath } throw "IDE $tag did not reach $what within $seconds seconds" } - # The driver types and pastes with a real keyboard. Two IDEs doing that together would - # steal each other's input. Start B only after A has submitted. A then waits at the - # provider barrier until B submits, so both delegations start together. + # The driver starts each task with a webview message and uses no keyboard. A background + # IDE window gets no native focus on Windows, so keyboard input would fail for IDE B. + # A waits at the provider barrier until B submits, so both delegations start together. Start-RaceIde 'A' - Wait-RaceIde 'A' { param($i) (Test-Path "$($i.Evidence)/progress.log") -and - (Select-String -Path "$($i.Evidence)/progress.log" -Pattern 'P-real-conversation-submitted' -Quiet) } 'its first submit' 300 Start-RaceIde 'B' foreach ($tag in 'A', 'B') { Wait-RaceIde $tag { param($i) Test-Path "$($i.Evidence)/status.txt" } 'a result' $DeadlineSeconds From d1e1d11d411bfc73ad8f0c7a4a500afd441a4332 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sun, 15 Nov 2026 20:29:07 +0000 Subject: [PATCH 3/5] test: address review findings for the history race test --- .github/workflows/windows-chat-focus.yml | 7 +- .../scripts/ui-harness/HarnessDriver.java | 3 +- jetbrains_plugin/scripts/ui-harness/README.md | 22 +- .../scripts/ui-harness/check_history_store.py | 22 +- .../scripts/ui-harness/run-history-race.sh | 198 ------------------ .../scripts/ui-harness/run-windows.ps1 | 4 + .../scripts/ui-harness/streaming_fixture.py | 2 +- .../ui-harness/test_check_history_store.py | 142 ++++++++++++- .../ui-harness/test_streaming_fixture.py | 18 +- 9 files changed, 200 insertions(+), 218 deletions(-) delete mode 100755 jetbrains_plugin/scripts/ui-harness/run-history-race.sh diff --git a/.github/workflows/windows-chat-focus.yml b/.github/workflows/windows-chat-focus.yml index 7d450b1..8fd14bc 100644 --- a/.github/workflows/windows-chat-focus.yml +++ b/.github/workflows/windows-chat-focus.yml @@ -69,7 +69,7 @@ jobs: sudo apt-get install --no-install-recommends build-essential libx11-dev libxkbfile-dev pkg-config - name: Validate the local streaming fixture and authenticated relay working-directory: jetbrains_plugin/scripts/ui-harness - run: python3 -m unittest -v test_streaming_fixture.py test_solheim_proxy.py + run: python3 -m unittest -v test_streaming_fixture.py test_solheim_proxy.py test_check_history_store.py - name: Prepare extension host source run: ./scripts/setup.sh --skip-deps --verbose - name: Cache the cross-platform VS Code runtime @@ -148,8 +148,9 @@ jobs: -EvidenceDir "$env:GITHUB_WORKSPACE/focus-evidence" # Issue #6. Two IDEs share ~/.zoo-code. Each one runs a parent task that # delegates to a subtask. The step fails if the shared history store has a - # shared index file, invalid items, wrong parent/child links, a subtask view - # that flips back, or an invalid status transition in the IDE logs. + # shared index file, invalid items, wrong parent/child links, more than three + # task switches in the webview, or an invalid status transition in the IDE logs. + # It does not detect a view that changes to the parent during the subtask and stays there. - name: Run two IDEs with parent tasks and subtasks against one history store if: ${{ !cancelled() }} run: | diff --git a/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java b/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java index fe346fb..6e8b544 100644 --- a/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java +++ b/jetbrains_plugin/scripts/ui-harness/HarnessDriver.java @@ -526,7 +526,8 @@ private void runRealConversation(Project project) throws Exception { /** * Waits until the parent task resumes and completes after its subtask. The webview shows * one task at a time. The expected task order is parent, child, parent. A fourth change - * of task means the view flipped back during the subtask. + * of task fails the test. The test counts task switches only. It cannot see a view + * that changes to the parent during the subtask and stays there until the parent resumes. */ private void waitForHistoryRace(Object webview, AtomicReference title) throws Exception { long deadline = System.currentTimeMillis() + 420_000; diff --git a/jetbrains_plugin/scripts/ui-harness/README.md b/jetbrains_plugin/scripts/ui-harness/README.md index 17b2300..803b70f 100644 --- a/jetbrains_plugin/scripts/ui-harness/README.md +++ b/jetbrains_plugin/scripts/ui-harness/README.md @@ -47,9 +47,9 @@ Record the exact ZIP, IDE version, and evidence directory when comparing builds. `run-windows.ps1 -HistoryRace` starts two IDEs with separate profiles and the same home directory, so both use one `~/.zoo-code/globalStorage`. Both IDEs start together. Each IDE starts a parent task with a webview `newTask` message. The test uses no keyboard, -because a background IDE window gets no native focus on Windows. The fixture makes the parent call `new_task`, holds both parents at a -barrier so the delegations overlap, streams both child tasks, and lets each parent -resume and finish. `check_history_store.py` then checks the store: +because a background IDE window gets no native focus on Windows. The fixture makes the +parent call `new_task`. It holds both parents at a barrier so the delegations overlap. +It then streams both child tasks and lets each parent resume and finish. `check_history_store.py` then checks the store: 1. No `_index.json` was written. 2. Every `history_item.json` is valid, has a real title, and has `size` above zero. @@ -59,7 +59,21 @@ resume and finish. `check_history_store.py` then checks the store: 6. The IDE logs hold no invalid status transition or failed persist message. The driver also fails if the webview changes task more than three times. The expected -order is parent, child, parent. The unit tests run without an IDE: +order is parent, child, parent. The test counts task switches only. It does not detect a +view that changes to the parent during the subtask and stays there until the parent resumes. + +The run uses the real user profile store, so the script refuses to start unless +`GITHUB_ACTIONS` is `true`. The Windows CI step has a 60 minute job timeout and uploads +the evidence as `windows-history-race--`. + +`-ExpectExtensionVersion` (Windows) and `EXPECT_EXTENSION_VERSION` (`run-solheim.sh`) make a run +fail when the Zoo Code extension in the plugin ZIP has another version. CI passes it to +the chat focus, shared history, and live Solheim steps. + +`streaming_fixture.py` flags: `--port`, `--log` (required), `--scenario history-race`, +`--chunks`, `--interval` (seconds between chunks), `--barrier` (parents to hold). + +The unit tests run without an IDE. Run them from this directory: `python3 -m unittest test_streaming_fixture test_check_history_store`. ## Linux live setup diff --git a/jetbrains_plugin/scripts/ui-harness/check_history_store.py b/jetbrains_plugin/scripts/ui-harness/check_history_store.py index 94f0c19..b76de99 100644 --- a/jetbrains_plugin/scripts/ui-harness/check_history_store.py +++ b/jetbrains_plugin/scripts/ui-harness/check_history_store.py @@ -84,9 +84,18 @@ def check_items(items, instances): def check_provider(path, instances): - events = [json.loads(line) for line in path.read_text(encoding='utf-8').splitlines() if line.strip()] - names = [e['event'] for e in events] - failures = [] + events, failures = [], [] + for number, line in enumerate(path.read_text(encoding='utf-8').splitlines(), 1): + if not line.strip(): + continue + try: + events.append(json.loads(line)) + except ValueError as error: + failures.append(f'{path.name}:{number}: provider log line is not valid JSON ({error})') + for event in events: + if not isinstance(event, dict): + failures.append(f'{path.name}: provider log line is not an object: {event!r}') + names = [e.get('event') for e in events if isinstance(e, dict)] if names.count('barrier-released') != instances: failures.append(f'barrier released {names.count("barrier-released")} times, expected {instances}') for bad in ('barrier-timeout', 'disconnected'): @@ -100,6 +109,9 @@ def check_provider(path, instances): def check_logs(paths): failures = [] for path in paths: + if not path.exists(): + failures.append(f'IDE log does not exist: {path}') + continue for number, line in enumerate(path.read_text(encoding='utf-8', errors='replace').splitlines(), 1): if BAD_LOG.search(line): failures.append(f'{path.name}:{number}: {line.strip()[:200]}') @@ -111,14 +123,14 @@ def main(): parser.add_argument('--storage', type=Path, required=True) parser.add_argument('--since', type=float, required=True, help='epoch seconds when the run started') parser.add_argument('--provider-log', type=Path, required=True) - parser.add_argument('--ide-log', type=Path, action='append', default=[]) + parser.add_argument('--ide-log', type=Path, action='append', required=True) parser.add_argument('--instances', type=int, default=2) parser.add_argument('--report', type=Path) args = parser.parse_args() items, failures = load_items(args.storage, args.since) failures += check_items(items, args.instances) failures += check_provider(args.provider_log, args.instances) - failures += check_logs([p for p in args.ide_log if p.exists()]) + failures += check_logs(args.ide_log) report = {'tasks': len(items), 'failures': failures} text = json.dumps(report, indent=2) if args.report: diff --git a/jetbrains_plugin/scripts/ui-harness/run-history-race.sh b/jetbrains_plugin/scripts/ui-harness/run-history-race.sh deleted file mode 100755 index 1c69df7..0000000 --- a/jetbrains_plugin/scripts/ui-harness/run-history-race.sh +++ /dev/null @@ -1,198 +0,0 @@ -#!/bin/bash -# Issue #6: run two IDEs against one shared Zoo Code history store (Linux, Xvfb). -# Each IDE runs a parent task that delegates to a subtask. check_history_store.py -# then checks the shared store. Both IDEs use a temporary user.home, so the test -# never touches the real ~/.zoo-code. -# -# Usage: IDE_DIR=/path/to/idea PLUGIN_ZIP=/path/to/ZooCode.zip bash run-history-race.sh -# Optional: HARNESS_WM_BIN (openbox), HARNESS_HEAP_MB, DEADLINE_SECONDS, EVIDENCE_DIR, -# EXPECT_EXTENSION_VERSION (fail when the Zoo Code extension in the ZIP differs; -# the ZIP name does not set the extension version) -set -uo pipefail -SOURCE_DIR="$(cd "$(dirname "$0")" && pwd)" -: "${IDE_DIR:?Set IDE_DIR to a JCEF-enabled IntelliJ installation}" -: "${PLUGIN_ZIP:?Set PLUGIN_ZIP to the built ZooCode distribution}" -IDE_DIR="$(realpath "$IDE_DIR")" -PLUGIN_ZIP="$(realpath "$PLUGIN_ZIP")" -PRODUCT_CODE="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["productCode"])' "$IDE_DIR/product-info.json")" -case "$PRODUCT_CODE" in - WS) IDE_LAUNCHER=webstorm ;; - IU|IC) IDE_LAUNCHER=idea ;; - *) echo "Unsupported harness IDE product: $PRODUCT_CODE" >&2; exit 1 ;; -esac -DEADLINE="${DEADLINE_SECONDS:-900}" -H="$(mktemp -d "${TMPDIR:-/tmp}/zoo-history-race.XXXXXXXX")" -EVIDENCE="${EVIDENCE_DIR:-$H/evidence}" -HOME_DIR="$H/home" -STORAGE="$HOME_DIR/.zoo-code/globalStorage" -echo "Disposable harness and evidence: $H" -mkdir -p "$H/driver/classes/META-INF" "$H/plugins-dist/harness-driver/lib" "$EVIDENCE" "$HOME_DIR" - -"$IDE_DIR/jbr/bin/javac" -cp "$IDE_DIR/lib/*" -d "$H/driver/classes" "$SOURCE_DIR/HarnessDriver.java" || exit 1 -cp "$SOURCE_DIR/plugin.xml" "$H/driver/classes/META-INF/plugin.xml" -# The IDE's JBR has javac but may omit the jar tool. A plugin jar is a zip. -python3 - "$H/driver/classes" "$H/plugins-dist/harness-driver/lib/driver.jar" <<'PY' || exit 1 -import pathlib, sys, zipfile -root = pathlib.Path(sys.argv[1]) -with zipfile.ZipFile(sys.argv[2], 'w', zipfile.ZIP_DEFLATED) as jar: - for path in sorted(root.rglob('*')): - if path.is_file(): - jar.write(path, path.relative_to(root).as_posix()) -PY -unzip -q "$PLUGIN_ZIP" -d "$H/plugins-dist" || exit 1 -[ -f "$H/plugins-dist/ZooCode/runtime/extension.js" ] || { echo 'Distribution has no release extension runtime' >&2; exit 1; } -EXTENSION_VERSION="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["version"])' "$H/plugins-dist/ZooCode/zoo-code/package.json")" || exit 1 -echo "$EXTENSION_VERSION" > "$EVIDENCE/extension-version.txt" -echo "Bundled Zoo Code extension version: $EXTENSION_VERSION" -if [ -n "${EXPECT_EXTENSION_VERSION:-}" ] && [ "$EXTENSION_VERSION" != "$EXPECT_EXTENSION_VERSION" ]; then - echo "Plugin ZIP bundles Zoo Code extension $EXTENSION_VERSION, expected $EXPECT_EXTENSION_VERSION." >&2 - rm -rf "$H/plugins-dist" "$H/driver" - exit 1 -fi - -PIDS=() -cleanup() { - for pid in "${PIDS[@]}"; do - pkill -TERM -P "$pid" 2>/dev/null - kill -TERM "$pid" 2>/dev/null - done - sleep 3 - for pid in "${PIDS[@]}"; do - pkill -9 -P "$pid" 2>/dev/null - kill -9 "$pid" 2>/dev/null - done - # Remove the large copies (plugins, IDE profiles, temporary home). The evidence stays, - # and it holds a copy of the history items. - rm -rf "$H/cfg" "$H/plugins-dist" "$H/driver" "$HOME_DIR" -} -trap cleanup EXIT - -# One virtual display for both IDEs. -XVFB_NUM=$((300 + RANDOM % 500)) -Xvfb ":$XVFB_NUM" -screen 0 1600x1000x24 -nolisten tcp > "$EVIDENCE/xvfb.log" 2>&1 & -PIDS+=($!) -sleep 2 -export DISPLAY=":$XVFB_NUM" -export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/xdg-$XVFB_NUM}" -mkdir -p "$XDG_RUNTIME_DIR"; chmod 700 "$XDG_RUNTIME_DIR" -if [ -n "${HARNESS_WM_BIN:-}" ]; then - "$HARNESS_WM_BIN" --sm-disable > "$EVIDENCE/window-manager.log" 2>&1 & - PIDS+=($!) -fi - -# Local provider with the history-race scenario. -python3 "$SOURCE_DIR/streaming_fixture.py" --scenario history-race --chunks 200 \ - --log "$EVIDENCE/provider.jsonl" > "$EVIDENCE/provider-stdout.log" 2> "$EVIDENCE/provider-stderr.log" & -PIDS+=($!) -for _ in $(seq 1 40); do - curl -fsS http://127.0.0.1:19487/health > /dev/null 2>&1 && break - sleep 0.25 -done -curl -fsS http://127.0.0.1:19487/health > /dev/null 2>&1 || { echo 'Local provider did not become healthy (is port 19487 busy?)' >&2; exit 1; } - -click_startup_dialogs() { - local ev="$1" - for _ in $(seq 1 30); do - [ -f "$ev/progress.log" ] && return - for wid in $(xdotool search --onlyvisible --name '(IntelliJ IDEA|WebStorm) User Agreement' 2>/dev/null); do - geom=$(xdotool getwindowgeometry --shell "$wid" 2>/dev/null) || continue - eval "$geom" - [ -f "$ev/progress.log" ] && return - if [ "${WIDTH:-0}" -ge 550 ] && [ "${WIDTH:-0}" -le 700 ] && [ "${HEIGHT:-0}" -ge 400 ] && [ "${HEIGHT:-0}" -le 520 ]; then - xdotool mousemove $((X + 40)) $((Y + HEIGHT * 85 / 100)) click 1 - sleep 1 - xdotool mousemove $((X + WIDTH * 89 / 100)) $((Y + HEIGHT * 93 / 100)) click 1 - sleep 2 - fi - done - sleep 3 - done -} - -start_ide() { - local tag="$1" - local cfg="$H/cfg/$tag" ev="$EVIDENCE/$tag" ws="$H/workspace-$tag" - mkdir -p "$cfg/config" "$cfg/system" "$cfg/log" "$ev" "$ws/src" "$H/props" - mkdir -p "$cfg/plugins" - cp -r "$H/plugins-dist/ZooCode" "$cfg/plugins/ZooCode" - cp -r "$H/plugins-dist/harness-driver" "$cfg/plugins/harness-driver" - printf 'public class Main { public static void main(String[] a) { System.out.println("throwaway"); } }\n' > "$ws/src/Main.java" - cat > "$H/props/idea-$tag.properties" <' - echo "-Dide.show.tips.on.startup.default.value=false" - echo "-Dide.browser.jcef.extra.args=--disable-dev-shm-usage" - echo "-Dzoo.harness.mode=P" - echo "-Dzoo.harness.fixture=true" - echo "-Dzoo.harness.historyRace=true" - echo "-Dzoo.harness.historyTag=$tag" - echo "-Dzoo.harness.evidence=$ev" - } > "$H/props/vm-$tag.vmoptions" - click_startup_dialogs "$ev" & - PIDS+=($!) - ( - export HOME="$HOME_DIR" - export IDEA_PROPERTIES="$H/props/idea-$tag.properties" IDEA_VM_OPTIONS="$H/props/vm-$tag.vmoptions" - export WEBIDE_PROPERTIES="$IDEA_PROPERTIES" WEBIDE_VM_OPTIONS="$IDEA_VM_OPTIONS" - cd "$IDE_DIR/bin" - exec "./$IDE_LAUNCHER" "$ws" - ) > "$ev/launcher-stdout.log" 2>&1 & - IDE_PIDS[$tag]=$! - PIDS+=($!) -} - -# Wait for a file test or a dead IDE. $1 tag, $2 description, $3 seconds, rest: test command. -wait_for() { - local tag="$1" what="$2" seconds="$3"; shift 3 - for _ in $(seq 1 "$seconds"); do - "$@" && return 0 - if ! kill -0 "${IDE_PIDS[$tag]}" 2>/dev/null; then echo "IDE $tag exited before $what" >&2; return 1; fi - sleep 1 - done - echo "IDE $tag did not reach $what within ${seconds}s" >&2 - return 1 -} -has_status() { [ -f "$EVIDENCE/$1/status.txt" ]; } - -declare -A IDE_PIDS -SINCE=$(( $(date +%s) - 2 )) -RESULT=FAIL - -# The driver starts each task with a webview message and uses no keyboard, so both IDEs -# can start together. A waits at the provider barrier until B submits, so both delegations -# start together. -start_ide A -start_ide B -ok=1 -for tag in A B; do - wait_for "$tag" 'a result' "$DEADLINE" has_status "$tag" || { ok=0; break; } - status="$(cat "$EVIDENCE/$tag/status.txt")" - [ "$status" = PASS ] || { echo "IDE $tag result: $status (see $EVIDENCE/$tag/evidence.json)" >&2; ok=0; break; } -done -if [ "$ok" = 1 ]; then - mkdir -p "$EVIDENCE/history-store" - find "$STORAGE" -name history_item.json -newermt "@$SINCE" -print0 2>/dev/null | while IFS= read -r -d '' f; do - cp "$f" "$EVIDENCE/history-store/$(basename "$(dirname "$f")").json" - done - if python3 "$SOURCE_DIR/check_history_store.py" --storage "$STORAGE" --since "$SINCE" \ - --provider-log "$EVIDENCE/provider.jsonl" --report "$EVIDENCE/history-check.json" \ - --ide-log "$H/cfg/A/log/idea.log" --ide-log "$H/cfg/B/log/idea.log"; then - RESULT=PASS - fi -fi - -for tag in A B; do cp "$H/cfg/$tag/log/idea.log" "$EVIDENCE/idea-$tag.log" 2>/dev/null || true; done -echo "HISTORY-RACE result: $RESULT" -echo "Evidence: $EVIDENCE" -[ "$RESULT" = PASS ] diff --git a/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 b/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 index 62fa97e..890247e 100644 --- a/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 +++ b/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 @@ -25,6 +25,10 @@ $ideProcess = $null $instances = @{} $fixtureProcess = $null $live = $Provider -eq 'solheim' +if ($HistoryRace -and $env:GITHUB_ACTIONS -ne 'true') { + # This mode shares the real user profile store. Copies of its tasks go to the evidence. + throw 'The history race test uses the real user profile. Run it only on a disposable CI runner.' +} if ($HistoryRace -and $live) { throw 'The history race test uses only the local fixture provider.' } $originalRelayToken = $env:SOLHEIM_RELAY_TOKEN $originalKeyFile = $env:SOLHEIM_KEY_FILE diff --git a/jetbrains_plugin/scripts/ui-harness/streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/streaming_fixture.py index 77fb92d..07d7af8 100644 --- a/jetbrains_plugin/scripts/ui-harness/streaming_fixture.py +++ b/jetbrains_plugin/scripts/ui-harness/streaming_fixture.py @@ -124,7 +124,7 @@ def emit(delta, finish=None): emit({}, 'tool_calls') self.wfile.write(b'data: [DONE]\n\n') self.wfile.flush() - self.server.record(event='completed', turn=turn, chunks=self.server.chunks) + self.server.record(event='completed', turn=turn, chunks=chunks) except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError): self.server.record(event='disconnected', turn=turn) diff --git a/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py b/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py index 9dc8a7e..ff2a459 100644 --- a/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py +++ b/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py @@ -19,7 +19,89 @@ def good_pair(tag, workspace): } +# Real items from Windows CI run 43562237505 (commit 5a494cf, IDEA 2026.2.1, extension 3.96.0). +# Only the temporary run folder name in `workspace` was shortened. +REAL_ITEMS = json.loads(r'''[ + { + "id": "01a1d898-c7ab-70da-936e-6467341e0125", + "parentTaskId": "01a1d898-bf28-71a1-92d9-df474bf72871", + "number": 1, + "ts": 1794635304779, + "task": "HISTORY_RACE_CHILD A reply with attempt_completion.", + "tokensIn": 0, + "tokensOut": 0, + "totalCost": 0, + "size": 10621, + "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-A", + "mode": "code", + "apiConfigName": "focus-fixture", + "status": "completed", + "completionResultSummary": "Fixture turn 4 complete." + }, + { + "id": "01a1d898-c663-72e9-b6dd-1caba4bda93a", + "parentTaskId": "01a1d898-4435-75fd-afc2-6ca3aa4c89a4", + "number": 1, + "ts": 1794635304331, + "task": "HISTORY_RACE_CHILD B reply with attempt_completion.", + "tokensIn": 0, + "tokensOut": 0, + "totalCost": 0, + "size": 10623, + "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-B", + "mode": "code", + "apiConfigName": "focus-fixture", + "status": "completed", + "completionResultSummary": "Fixture turn 3 complete." + }, + { + "id": "01a1d898-bf28-71a1-92d9-df474bf72871", + "number": 1, + "ts": 1794635306508, + "task": "HISTORY_RACE_PARENT A delegate one subtask, then finish.", + "tokensIn": 0, + "tokensOut": 0, + "totalCost": 0, + "size": 7694, + "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-A", + "mode": "code", + "apiConfigName": "focus-fixture", + "status": "active", + "childIds": [ + "01a1d898-c7ab-70da-936e-6467341e0125" + ], + "completedByChildId": "01a1d898-c7ab-70da-936e-6467341e0125", + "completionResultSummary": "Fixture turn 4 complete." + }, + { + "id": "01a1d898-4435-75fd-afc2-6ca3aa4c89a4", + "number": 1, + "ts": 1794635306029, + "task": "HISTORY_RACE_PARENT B delegate one subtask, then finish.", + "tokensIn": 0, + "tokensOut": 0, + "totalCost": 0, + "size": 7698, + "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-B", + "mode": "code", + "apiConfigName": "focus-fixture", + "status": "active", + "childIds": [ + "01a1d898-c663-72e9-b6dd-1caba4bda93a" + ], + "completedByChildId": "01a1d898-c663-72e9-b6dd-1caba4bda93a", + "completionResultSummary": "Fixture turn 3 complete." + } +]''') + + class CheckItemsTest(unittest.TestCase): + def test_real_extension_items_pass(self): + items = {i['id']: i for i in REAL_ITEMS} + self.assertEqual(4, len(items)) + self.assertEqual([], check_items(items, 2)) + + def two(self): return {**good_pair('A', 'wa'), **good_pair('B', 'wb')} @@ -50,6 +132,38 @@ def test_stuck_delegation_and_open_child_fail(self): self.assertTrue(any('still waits' in f for f in failures)) self.assertTrue(any('child status' in f for f in failures)) + def fails(self, items, message): + self.assertIn(message, check_items(items, 2)) + + def test_each_bad_item_branch_reports_an_exact_message(self): + items = self.two() + items['pA']['task'] = 'Work #2' + self.fails(items, "pA: placeholder or empty title 'Work #2'") + items = self.two() + items['pA']['task'] = ' ' + self.fails(items, "pA: placeholder or empty title ' '") + items = self.two() + items['pA']['task'] = 'no marker' + self.fails(items, "pA: title does not carry a run marker: 'no marker'") + items = self.two() + items['pA']['delegatedToId'] = 'cA' + self.fails(items, 'pA: parent still waits on a child (status \'active\')') + items = self.two() + items['pA']['awaitingChildId'] = 'cA' + self.fails(items, 'pA: parent still waits on a child (status \'active\')') + items = self.two() + items['cA']['workspace'] = 'other' + self.fails(items, 'pA: parent and child workspaces differ') + + def test_shared_workspace_and_extra_child_fail(self): + items = {**good_pair('A', 'w'), **good_pair('B', 'w')} + self.assertTrue(any('distinct workspaces' in f for f in check_items(items, 2))) + items = self.two() + items.update(good_pair('A', 'wa')) + items['cA2'] = item('cA2', 'HISTORY_RACE_CHILD A again', parentTaskId='pA', + status='completed', workspace='wa') + self.fails(items, 'pA: expected one child for tag A, found 2') + def test_missing_task_fails(self): items = self.two() del items['cB'] @@ -67,6 +181,16 @@ def test_index_file_and_bad_json_fail(self): self.assertEqual({}, items) self.assertEqual(2, len(failures)) + def test_id_mismatch_and_non_object_fail(self): + with tempfile.TemporaryDirectory() as directory: + tasks = Path(directory) / 'ext' / 'tasks' + for name, text in (('x', '{"id": "y"}'), ('z', '[1]')): + (tasks / name).mkdir(parents=True) + (tasks / name / 'history_item.json').write_text(text) + items, failures = load_items(Path(directory), 0) + self.assertEqual({}, items) + self.assertEqual(["x: id field is 'y'", 'z: history_item.json is not an object'], failures) + def test_old_tasks_are_ignored(self): with tempfile.TemporaryDirectory() as directory: task = Path(directory) / 'ext' / 'tasks' / 'x' @@ -77,11 +201,25 @@ def test_old_tasks_are_ignored(self): def test_provider_and_log_checks(self): with tempfile.TemporaryDirectory() as directory: provider = Path(directory) / 'p.jsonl' - events = ['barrier-released'] * 2 + ['completed'] * 6 - provider.write_text('\n'.join(json.dumps({'event': e}) for e in events)) + good = ['barrier-released'] * 2 + ['completed'] * 6 + provider.write_text('\n'.join(json.dumps({'event': e}) for e in good)) self.assertEqual([], check_provider(provider, 2)) provider.write_text(json.dumps({'event': 'barrier-timeout'})) self.assertTrue(check_provider(provider, 2)) + def provider_failures(events): + provider.write_text('\n'.join(json.dumps({'event': e}) for e in events)) + return check_provider(provider, 2) + self.assertIn('provider logged disconnected', provider_failures(good + ['disconnected'])) + self.assertIn('barrier released 1 times, expected 2', + provider_failures(['barrier-released'] + ['completed'] * 6)) + self.assertIn('5 completed streams, expected 6', + provider_failures(['barrier-released'] * 2 + ['completed'] * 5)) + provider.write_text('{"event": "completed"}\n{broken') + self.assertTrue(any('not valid JSON' in f for f in check_provider(provider, 2))) + provider.write_text('[1]\n' + '\n'.join(json.dumps({'event': e}) for e in good)) + self.assertEqual(["p.jsonl: provider log line is not an object: [1]"], check_provider(provider, 2)) + missing = Path(directory) / 'missing.log' + self.assertEqual([f'IDE log does not exist: {missing}'], check_logs([missing])) log = Path(directory) / 'idea.log' log.write_text('ok\nInvalid status transition interrupted -> delegated\n') self.assertEqual(1, len(check_logs([log]))) diff --git a/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py index edf0695..0a825e6 100644 --- a/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py +++ b/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py @@ -131,11 +131,20 @@ def call(tag, text): results[tag] = [json.loads(line[6:]) for line in data.splitlines() if line.startswith('data: {')] connection.close() - parents = [threading.Thread(target=call, args=(tag, f'HISTORY_RACE_PARENT {tag}')) for tag in 'AB'] - for parent in parents: - parent.start() - for parent in parents: + def events(): + return [json.loads(line)['event'] for line in log.read_text().splitlines()] if log.exists() else [] + + parents = {tag: threading.Thread(target=call, args=(tag, f'HISTORY_RACE_PARENT {tag}')) for tag in 'AB'} + # Parent A alone must stay blocked: no new_task and no barrier-released. + parents['A'].start() + parents['A'].join(0.5) + self.assertTrue(parents['A'].is_alive()) + self.assertNotIn('A', results) + self.assertNotIn('barrier-released', events()) + parents['B'].start() + for parent in parents.values(): parent.join(15) + self.assertFalse(parent.is_alive()) for tag in 'AB': call_delta = results[tag][-2]['choices'][0]['delta']['tool_calls'][0]['function'] self.assertEqual('new_task', call_delta['name']) @@ -169,6 +178,7 @@ def test_lone_parent_times_out_instead_of_passing(self): connection.close() events = [json.loads(line)['event'] for line in log.read_text().splitlines()] self.assertIn('barrier-timeout', events) + self.assertNotIn('barrier-released', events) if __name__ == '__main__': From eddce2b2e5b51b51bcee4e47c813ec5ee240be4b Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sun, 15 Nov 2026 21:51:55 +0000 Subject: [PATCH 4/5] test: close review gaps in the history race test --- jetbrains_plugin/scripts/ui-harness/README.md | 8 +- .../ui-harness/persist/check_history_store.py | 143 +++++++++++ .../ui-harness/persist/solheim_proxy.py | 145 +++++++++++ .../ui-harness/persist/streaming_fixture.py | 157 ++++++++++++ .../persist/test_check_history_store.py | 235 ++++++++++++++++++ .../ui-harness/persist/test_solheim_proxy.py | 156 ++++++++++++ .../persist/test_streaming_fixture.py | 188 ++++++++++++++ .../re-delegate/check_history_store.py | 143 +++++++++++ .../ui-harness/re-delegate/solheim_proxy.py | 145 +++++++++++ .../re-delegate/streaming_fixture.py | 157 ++++++++++++ .../re-delegate/test_check_history_store.py | 235 ++++++++++++++++++ .../re-delegate/test_solheim_proxy.py | 156 ++++++++++++ .../re-delegate/test_streaming_fixture.py | 188 ++++++++++++++ .../scripts/ui-harness/run-windows.ps1 | 8 +- .../ui-harness/test_check_history_store.py | 6 + .../ui-harness/test_streaming_fixture.py | 10 +- 16 files changed, 2070 insertions(+), 10 deletions(-) create mode 100644 jetbrains_plugin/scripts/ui-harness/persist/check_history_store.py create mode 100644 jetbrains_plugin/scripts/ui-harness/persist/solheim_proxy.py create mode 100644 jetbrains_plugin/scripts/ui-harness/persist/streaming_fixture.py create mode 100644 jetbrains_plugin/scripts/ui-harness/persist/test_check_history_store.py create mode 100644 jetbrains_plugin/scripts/ui-harness/persist/test_solheim_proxy.py create mode 100644 jetbrains_plugin/scripts/ui-harness/persist/test_streaming_fixture.py create mode 100644 jetbrains_plugin/scripts/ui-harness/re-delegate/check_history_store.py create mode 100644 jetbrains_plugin/scripts/ui-harness/re-delegate/solheim_proxy.py create mode 100644 jetbrains_plugin/scripts/ui-harness/re-delegate/streaming_fixture.py create mode 100644 jetbrains_plugin/scripts/ui-harness/re-delegate/test_check_history_store.py create mode 100644 jetbrains_plugin/scripts/ui-harness/re-delegate/test_solheim_proxy.py create mode 100644 jetbrains_plugin/scripts/ui-harness/re-delegate/test_streaming_fixture.py diff --git a/jetbrains_plugin/scripts/ui-harness/README.md b/jetbrains_plugin/scripts/ui-harness/README.md index 803b70f..0f319de 100644 --- a/jetbrains_plugin/scripts/ui-harness/README.md +++ b/jetbrains_plugin/scripts/ui-harness/README.md @@ -63,12 +63,12 @@ order is parent, child, parent. The test counts task switches only. It does not view that changes to the parent during the subtask and stays there until the parent resumes. The run uses the real user profile store, so the script refuses to start unless -`GITHUB_ACTIONS` is `true`. The Windows CI step has a 60 minute job timeout and uploads +`GITHUB_ACTIONS` is `true`. The Windows CI job has a 60 minute timeout and uploads the evidence as `windows-history-race--`. -`-ExpectExtensionVersion` (Windows) and `EXPECT_EXTENSION_VERSION` (`run-solheim.sh`) make a run -fail when the Zoo Code extension in the plugin ZIP has another version. CI passes it to -the chat focus, shared history, and live Solheim steps. +`-ExpectExtensionVersion` (`run-windows.ps1`) makes a run fail when the Zoo Code extension +in the plugin ZIP has another version. CI passes it to the chat focus, shared history, +and live Solheim steps. `streaming_fixture.py` flags: `--port`, `--log` (required), `--scenario history-race`, `--chunks`, `--interval` (seconds between chunks), `--barrier` (parents to hold). diff --git a/jetbrains_plugin/scripts/ui-harness/persist/check_history_store.py b/jetbrains_plugin/scripts/ui-harness/persist/check_history_store.py new file mode 100644 index 0000000..b76de99 --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/persist/check_history_store.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Check the shared Zoo Code task history after a concurrent two-IDE run. + +Issue #6: several IDE processes write one global history store. A safe store +keeps one valid history_item.json per task, with no shared index file and with +parent and child links that stay inside one IDE's task tree. +""" +import argparse +import json +import re +import sys +from pathlib import Path + +PLACEHOLDER = re.compile(r'^\s*(Task|Work) #\d+') +BAD_LOG = re.compile(r'invalid status transition|DeltaRejectedError|failed to persist|Cannot re-delegate', re.I) +MARKER = re.compile(r'HISTORY_RACE_(PARENT|CHILD) (\w+)') + + +def load_items(storage, since): + """Return ({task id: item}, failures) for tasks written at or after `since`.""" + items, failures = {}, [] + for index in storage.glob('**/tasks/_index.json'): + if index.stat().st_mtime >= since: + failures.append(f'shared index file was written: {index}') + for path in sorted(storage.glob('**/tasks/*/history_item.json')): + if path.stat().st_mtime < since: + continue + try: + item = json.loads(path.read_text(encoding='utf-8')) + except (OSError, ValueError) as error: + failures.append(f'{path.parent.name}: history_item.json is not valid JSON ({error})') + continue + if not isinstance(item, dict): + failures.append(f'{path.parent.name}: history_item.json is not an object') + continue + if item.get('id') != path.parent.name: + failures.append(f'{path.parent.name}: id field is {item.get("id")!r}') + continue + items[item['id']] = item + return items, failures + + +def check_items(items, instances): + failures = [] + tasks = {} + for task_id, item in items.items(): + text = item.get('task') or '' + if PLACEHOLDER.match(text) or not text.strip(): + failures.append(f'{task_id}: placeholder or empty title {text!r}') + continue + match = MARKER.search(text) + if not match: + failures.append(f'{task_id}: title does not carry a run marker: {text!r}') + continue + if not item.get('size'): + failures.append(f'{task_id}: size is {item.get("size")!r} after real work') + tasks[task_id] = (match.group(1), match.group(2), item) + parents = {i: t for i, t in tasks.items() if t[0] == 'PARENT'} + children = {i: t for i, t in tasks.items() if t[0] == 'CHILD'} + if len(parents) != instances or len(children) != instances: + failures.append(f'expected {instances} parents and {instances} children, ' + f'found {len(parents)} and {len(children)}') + for parent_id, (_, tag, parent) in parents.items(): + own = [c for c, t in children.items() if t[1] == tag] + if len(own) != 1: + failures.append(f'{parent_id}: expected one child for tag {tag}, found {len(own)}') + continue + child_id = own[0] + child = children[child_id][2] + if parent.get('childIds') != [child_id]: + failures.append(f'{parent_id}: childIds {parent.get("childIds")!r}, expected {[child_id]!r}') + if child.get('parentTaskId') != parent_id: + failures.append(f'{child_id}: parentTaskId {child.get("parentTaskId")!r}, expected {parent_id!r}') + if child.get('status') != 'completed': + failures.append(f'{child_id}: child status {child.get("status")!r}, expected completed') + if parent.get('status') == 'delegated' or parent.get('awaitingChildId') or parent.get('delegatedToId'): + failures.append(f'{parent_id}: parent still waits on a child (status {parent.get("status")!r})') + if parent.get('workspace') != child.get('workspace'): + failures.append(f'{parent_id}: parent and child workspaces differ') + workspaces = {t[2].get('workspace') for t in parents.values()} + if instances > 1 and len(workspaces) != instances: + failures.append(f'expected {instances} distinct workspaces, found {sorted(map(str, workspaces))}') + return failures + + +def check_provider(path, instances): + events, failures = [], [] + for number, line in enumerate(path.read_text(encoding='utf-8').splitlines(), 1): + if not line.strip(): + continue + try: + events.append(json.loads(line)) + except ValueError as error: + failures.append(f'{path.name}:{number}: provider log line is not valid JSON ({error})') + for event in events: + if not isinstance(event, dict): + failures.append(f'{path.name}: provider log line is not an object: {event!r}') + names = [e.get('event') for e in events if isinstance(e, dict)] + if names.count('barrier-released') != instances: + failures.append(f'barrier released {names.count("barrier-released")} times, expected {instances}') + for bad in ('barrier-timeout', 'disconnected'): + if bad in names: + failures.append(f'provider logged {bad}') + if names.count('completed') != 3 * instances: + failures.append(f'{names.count("completed")} completed streams, expected {3 * instances}') + return failures + + +def check_logs(paths): + failures = [] + for path in paths: + if not path.exists(): + failures.append(f'IDE log does not exist: {path}') + continue + for number, line in enumerate(path.read_text(encoding='utf-8', errors='replace').splitlines(), 1): + if BAD_LOG.search(line): + failures.append(f'{path.name}:{number}: {line.strip()[:200]}') + return failures + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--storage', type=Path, required=True) + parser.add_argument('--since', type=float, required=True, help='epoch seconds when the run started') + parser.add_argument('--provider-log', type=Path, required=True) + parser.add_argument('--ide-log', type=Path, action='append', required=True) + parser.add_argument('--instances', type=int, default=2) + parser.add_argument('--report', type=Path) + args = parser.parse_args() + items, failures = load_items(args.storage, args.since) + failures += check_items(items, args.instances) + failures += check_provider(args.provider_log, args.instances) + failures += check_logs(args.ide_log) + report = {'tasks': len(items), 'failures': failures} + text = json.dumps(report, indent=2) + if args.report: + args.report.write_text(text, encoding='utf-8') + print(text) + return 1 if failures else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/jetbrains_plugin/scripts/ui-harness/persist/solheim_proxy.py b/jetbrains_plugin/scripts/ui-harness/persist/solheim_proxy.py new file mode 100644 index 0000000..669102d --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/persist/solheim_proxy.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Loopback authentication relay for manual Solheim extension focus smoke tests. + +Real credentials stay in SOLHEIM_KEY_FILE (mode 0600), outside IDE settings/logs. +The extension uses a per-run relay token and http://127.0.0.1:19487/v1. Logs contain only +request metadata, upstream status, stream byte counts, and tool names. +""" +import http.server +import hmac +import json +import os +import time +import urllib.error +import urllib.request +from pathlib import Path + +KEY = None +TOKEN = None +BASE = 'https://api.solheim.ai/v1' +MAX_BODY_BYTES = 8 * 1024 * 1024 +LOG = Path(os.environ.get('SOLHEIM_PROXY_LOG', '/tmp/zoo-solheim-proxy.jsonl')) + + +def record(**fields): + with LOG.open('a') as output: + output.write(json.dumps({'time': time.time(), **fields}) + '\n') + + +class RefuseRedirects(urllib.request.HTTPRedirectHandler): + def redirect_request(self, request, response, code, message, headers, new_url): + # Never send provider credentials to a redirected URL. + return None + + +UPSTREAM = urllib.request.build_opener(RefuseRedirects()) + + +class Relay(http.server.BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def log_message(self, *_): + pass + + def do_GET(self): + self.relay() + + def do_POST(self): + self.relay() + + def relay(self): + expected_host = '127.0.0.1:' + str(self.server.server_port) + if self.headers.get('Host') != expected_host or self.headers.get('Origin') is not None: + self.send_error(403) + return + supplied = self.headers.get('Authorization', '') + if not TOKEN or not hmac.compare_digest(supplied, 'Bearer ' + TOKEN): + self.send_error(401) + return + if not self.path.startswith('/v1/'): + self.send_error(404) + return + if self.command == 'GET' and self.path == '/v1/health': + # Local readiness only; never probe the paid upstream in a startup loop. + self.send_response(200) + self.send_header('Content-Length', '5') + self.end_headers() + self.wfile.write(b'ready') + return + lengths = self.headers.get_all('Content-Length', []) + length_text = lengths[0] if lengths else '0' + if (len(lengths) > 1 or self.headers.get('Transfer-Encoding') is not None + or not length_text.isascii() or not length_text.isdecimal()): + self.send_error(400, 'Invalid request framing') + return + # Check before int conversion, including absurdly long decimal strings. + if len(length_text.lstrip('0')) > 8 or int(length_text.lstrip('0') or '0') > MAX_BODY_BYTES: + self.send_error(413, 'Request body too large') + return + length = int(length_text.lstrip('0') or '0') + body = self.rfile.read(length) + try: + parsed = json.loads(body) if body else {} + if len(body) != length or not isinstance(parsed, dict): + raise ValueError('Expected a JSON object') + if not isinstance(parsed.get('messages', []), list): + raise ValueError('Expected a message list') + except (ValueError, UnicodeError): + self.send_error(400, 'Invalid JSON request body') + return + record(event='request', path=self.path, model=parsed.get('model'), + messages=len(parsed.get('messages', [])), stream=parsed.get('stream', False)) + request = urllib.request.Request(BASE + self.path[3:], data=body or None, + method=self.command, + headers={'Authorization': 'Bearer ' + KEY, + 'Content-Type': 'application/json'}) + try: + response = UPSTREAM.open(request, timeout=180) + except urllib.error.HTTPError as error: + response = error + except Exception as error: + record(event='transport-error', error=type(error).__name__) + self.send_error(502) + return + self.send_response(response.status) + self.send_header('Content-Type', response.headers.get('Content-Type', 'application/json')) + self.send_header('Connection', 'close') + self.end_headers() + count = 0 + buffer = b'' + tools = set() + try: + while True: + chunk = response.read1(4096) + if not chunk: + break + count += len(chunk) + self.wfile.write(chunk) + self.wfile.flush() + buffer += chunk + while b'\n' in buffer: + line, buffer = buffer.split(b'\n', 1) + if line.startswith(b'data: ') and line != b'data: [DONE]': + try: + event = json.loads(line[6:]) + for choice in event.get('choices', []): + for tool in choice.get('delta', {}).get('tool_calls', []): + name = tool.get('function', {}).get('name') + if name: + tools.add(name) + except (ValueError, TypeError): + pass + except (BrokenPipeError, ConnectionResetError): + record(event='client-disconnected') + finally: + response.close() + self.close_connection = True + record(event='response', status=response.status, bytes=count, tools=sorted(tools)) + + +if __name__ == '__main__': + KEY = Path(os.environ['SOLHEIM_KEY_FILE']).read_text().strip() + TOKEN = os.environ['SOLHEIM_RELAY_TOKEN'] + if len(TOKEN) < 32: + raise SystemExit('Use a random relay token of at least 32 characters') + http.server.ThreadingHTTPServer(('127.0.0.1', 19487), Relay).serve_forever() diff --git a/jetbrains_plugin/scripts/ui-harness/persist/streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/persist/streaming_fixture.py new file mode 100644 index 0000000..07d7af8 --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/persist/streaming_fixture.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Deterministic loopback OpenAI fixture for headful chat-focus CI tests. + +Only the provider is simulated. The extension, React history, JCEF, and native +keyboard delivery remain real. No commands or file edits are requested. +""" +import argparse +import http.server +import json +import re +import threading +import time +from pathlib import Path + + +def _text(content): + if isinstance(content, list): + return ''.join(part.get('text', '') for part in content if isinstance(part, dict)) + return content if isinstance(content, str) else '' + + +def history_role(messages): + """Classify a request in the history-race scenario. + + The first user message names the task: a parent carries HISTORY_RACE_PARENT + and a child carries HISTORY_RACE_CHILD . A parent that already holds a + new_task call in a non-system message has delegated, so it resumes. + """ + users = [m for m in messages if m.get('role') == 'user'] + first = _text(users[0].get('content')) if users else '' + child = re.search(r'HISTORY_RACE_CHILD (\w+)', first) + if child: + return 'child', child.group(1) + parent = re.search(r'HISTORY_RACE_PARENT (\w+)', first) + if not parent: + return 'plain', '' + body = json.dumps([m for m in messages if m.get('role') != 'system']) + return ('resume' if 'new_task' in body else 'delegate'), parent.group(1) + + +class Fixture(http.server.BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_GET(self): + if self.path == '/health': + self.send_response(200) + self.end_headers() + self.wfile.write(b'ready') + elif self.path == '/v1/models': + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps({'object': 'list', 'data': [ + {'id': 'focus-fixture', 'object': 'model', 'owned_by': 'fixture'} + ]}).encode()) + else: + self.send_error(404) + + def do_POST(self): + if self.path != '/v1/chat/completions': + self.send_error(404) + return + if self.headers.get('Authorization') != 'Bearer focus-fixture-local-only': + self.send_error(401) + return + try: + length = int(self.headers.get('Content-Length', '0')) + if not 0 < length <= 8 * 1024 * 1024: + raise ValueError('body size') + request = json.loads(self.rfile.read(length)) + if request.get('stream') is not True: + raise ValueError('streaming required') + tools = [t.get('function', {}).get('name') for t in request.get('tools', [])] + if 'attempt_completion' not in tools: + raise ValueError('native attempt_completion tool required') + except (ValueError, TypeError, AttributeError): + self.send_error(400, 'Expected streaming request with native completion tool') + return + with self.server.counter_lock: + self.server.requests += 1 + turn = self.server.requests + role, tag = 'plain', '' + if self.server.scenario == 'history-race': + role, tag = history_role(request.get('messages', [])) + self.server.record(event='request', turn=turn, tools=tools, role=role, tag=tag) + self.send_response(200) + self.send_header('Content-Type', 'text/event-stream') + self.send_header('Cache-Control', 'no-cache') + self.end_headers() + + def emit(delta, finish=None): + chunk = {'id': f'focus-{turn}', 'object': 'chat.completion.chunk', + 'created': int(time.time()), 'model': 'focus-fixture', + 'choices': [{'index': 0, 'delta': delta, 'finish_reason': finish}]} + self.wfile.write(('data: ' + json.dumps(chunk) + '\n\n').encode()) + self.wfile.flush() + + try: + emit({'role': 'assistant', 'content': ''}) + chunks = self.server.chunks + if role == 'delegate': + # Hold every parent until all IDEs have submitted, so the delegations + # and the child streams that follow overlap in time. + try: + self.server.barrier.wait() + except threading.BrokenBarrierError: + self.server.record(event='barrier-timeout', turn=turn, tag=tag) + return + self.server.record(event='barrier-released', turn=turn, tag=tag) + if role in ('delegate', 'resume'): + chunks = 3 + for index in range(chunks): + emit({'content': f'History update {index + 1} for turn {turn}. '}) + time.sleep(self.server.interval) + if role == 'delegate': + call = ('new_task', {'mode': 'code', 'todos': None, + 'message': f'HISTORY_RACE_CHILD {tag} reply with attempt_completion.'}) + else: + call = ('attempt_completion', {'result': f'Fixture turn {turn} complete.'}) + emit({'tool_calls': [{'index': 0, 'id': f'call-{turn}', + 'type': 'function', 'function': { + 'name': call[0], 'arguments': json.dumps(call[1])}}]}) + emit({}, 'tool_calls') + self.wfile.write(b'data: [DONE]\n\n') + self.wfile.flush() + self.server.record(event='completed', turn=turn, chunks=chunks) + except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError): + self.server.record(event='disconnected', turn=turn) + + +def make_server(port=19487, log=None, chunks=100, interval=0.25, scenario=None, barrier=2, barrier_timeout=300): + server = http.server.ThreadingHTTPServer(('127.0.0.1', port), Fixture) + server.requests = 0 + server.counter_lock = threading.Lock() + server.chunks, server.interval = chunks, interval + server.scenario = scenario + server.barrier = threading.Barrier(barrier, timeout=barrier_timeout) + + def record(**fields): + if log: + with server.counter_lock, Path(log).open('a', encoding='utf-8') as output: + output.write(json.dumps({'time': time.time(), **fields}) + '\n') + server.record = record + return server + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--port', type=int, default=19487) + parser.add_argument('--log', type=Path, required=True) + parser.add_argument('--scenario', choices=['history-race']) + parser.add_argument('--chunks', type=int, default=100) + parser.add_argument('--interval', type=float, default=0.25) + parser.add_argument('--barrier', type=int, default=2) + args = parser.parse_args() + make_server(args.port, args.log, args.chunks, args.interval, args.scenario, args.barrier).serve_forever() diff --git a/jetbrains_plugin/scripts/ui-harness/persist/test_check_history_store.py b/jetbrains_plugin/scripts/ui-harness/persist/test_check_history_store.py new file mode 100644 index 0000000..b095026 --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/persist/test_check_history_store.py @@ -0,0 +1,235 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from check_history_store import check_items, check_logs, check_provider, load_items + + +def item(task_id, text, **extra): + return {'id': task_id, 'task': text, 'size': 10, 'workspace': 'w', **extra} + + +def good_pair(tag, workspace): + return { + 'p' + tag: item('p' + tag, f'HISTORY_RACE_PARENT {tag} go', childIds=['c' + tag], + status='active', workspace=workspace), + 'c' + tag: item('c' + tag, f'HISTORY_RACE_CHILD {tag} go', parentTaskId='p' + tag, + status='completed', workspace=workspace), + } + + +# Real items from Windows CI run 43562237505 (commit 5a494cf, IDEA 2026.2.1, extension 3.96.0). +# Only the temporary run folder name in `workspace` was shortened. +REAL_ITEMS = json.loads(r'''[ + { + "id": "01a1d898-c7ab-70da-936e-6467341e0125", + "parentTaskId": "01a1d898-bf28-71a1-92d9-df474bf72871", + "number": 1, + "ts": 1794635304779, + "task": "HISTORY_RACE_CHILD A reply with attempt_completion.", + "tokensIn": 0, + "tokensOut": 0, + "totalCost": 0, + "size": 10621, + "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-A", + "mode": "code", + "apiConfigName": "focus-fixture", + "status": "completed", + "completionResultSummary": "Fixture turn 4 complete." + }, + { + "id": "01a1d898-c663-72e9-b6dd-1caba4bda93a", + "parentTaskId": "01a1d898-4435-75fd-afc2-6ca3aa4c89a4", + "number": 1, + "ts": 1794635304331, + "task": "HISTORY_RACE_CHILD B reply with attempt_completion.", + "tokensIn": 0, + "tokensOut": 0, + "totalCost": 0, + "size": 10623, + "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-B", + "mode": "code", + "apiConfigName": "focus-fixture", + "status": "completed", + "completionResultSummary": "Fixture turn 3 complete." + }, + { + "id": "01a1d898-bf28-71a1-92d9-df474bf72871", + "number": 1, + "ts": 1794635306508, + "task": "HISTORY_RACE_PARENT A delegate one subtask, then finish.", + "tokensIn": 0, + "tokensOut": 0, + "totalCost": 0, + "size": 7694, + "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-A", + "mode": "code", + "apiConfigName": "focus-fixture", + "status": "active", + "childIds": [ + "01a1d898-c7ab-70da-936e-6467341e0125" + ], + "completedByChildId": "01a1d898-c7ab-70da-936e-6467341e0125", + "completionResultSummary": "Fixture turn 4 complete." + }, + { + "id": "01a1d898-4435-75fd-afc2-6ca3aa4c89a4", + "number": 1, + "ts": 1794635306029, + "task": "HISTORY_RACE_PARENT B delegate one subtask, then finish.", + "tokensIn": 0, + "tokensOut": 0, + "totalCost": 0, + "size": 7698, + "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-B", + "mode": "code", + "apiConfigName": "focus-fixture", + "status": "active", + "childIds": [ + "01a1d898-c663-72e9-b6dd-1caba4bda93a" + ], + "completedByChildId": "01a1d898-c663-72e9-b6dd-1caba4bda93a", + "completionResultSummary": "Fixture turn 3 complete." + } +]''') + + +class CheckItemsTest(unittest.TestCase): + def test_real_extension_items_pass(self): + items = {i['id']: i for i in REAL_ITEMS} + self.assertEqual(4, len(items)) + self.assertEqual([], check_items(items, 2)) + + + def two(self): + return {**good_pair('A', 'wa'), **good_pair('B', 'wb')} + + def test_clean_store_passes(self): + self.assertEqual([], check_items(self.two(), 2)) + + def test_placeholder_title_and_zero_size_fail(self): + items = self.two() + items['pA']['task'] = 'Task #1' + items['cB']['size'] = 0 + failures = check_items(items, 2) + self.assertTrue(any('placeholder' in f for f in failures)) + self.assertTrue(any('size is 0' in f for f in failures)) + + def test_cross_linked_children_fail(self): + items = self.two() + items['pA']['childIds'] = ['cB'] + items['cA']['parentTaskId'] = 'pB' + failures = check_items(items, 2) + self.assertTrue(any('childIds' in f for f in failures)) + self.assertTrue(any('parentTaskId' in f for f in failures)) + + def test_stuck_delegation_and_open_child_fail(self): + items = self.two() + items['pA'].update(status='delegated', awaitingChildId='cA') + items['cA']['status'] = 'interrupted' + failures = check_items(items, 2) + self.assertTrue(any('still waits' in f for f in failures)) + self.assertTrue(any('child status' in f for f in failures)) + + def fails(self, items, message): + self.assertIn(message, check_items(items, 2)) + + def test_each_bad_item_branch_reports_an_exact_message(self): + items = self.two() + items['pA']['task'] = 'Work #2' + self.fails(items, "pA: placeholder or empty title 'Work #2'") + items = self.two() + items['pA']['task'] = ' ' + self.fails(items, "pA: placeholder or empty title ' '") + items = self.two() + items['pA']['task'] = 'no marker' + self.fails(items, "pA: title does not carry a run marker: 'no marker'") + items = self.two() + items['pA']['delegatedToId'] = 'cA' + self.fails(items, 'pA: parent still waits on a child (status \'active\')') + items = self.two() + items['pA']['awaitingChildId'] = 'cA' + self.fails(items, 'pA: parent still waits on a child (status \'active\')') + items = self.two() + items['cA']['workspace'] = 'other' + self.fails(items, 'pA: parent and child workspaces differ') + + def test_shared_workspace_and_extra_child_fail(self): + items = {**good_pair('A', 'w'), **good_pair('B', 'w')} + self.assertTrue(any('distinct workspaces' in f for f in check_items(items, 2))) + items = self.two() + items.update(good_pair('A', 'wa')) + items['cA2'] = item('cA2', 'HISTORY_RACE_CHILD A again', parentTaskId='pA', + status='completed', workspace='wa') + self.fails(items, 'pA: expected one child for tag A, found 2') + + def test_missing_task_fails(self): + items = self.two() + del items['cB'] + self.assertTrue(any('expected 2 parents' in f for f in check_items(items, 2))) + + +class StoreAndLogTest(unittest.TestCase): + def test_index_file_and_bad_json_fail(self): + with tempfile.TemporaryDirectory() as directory: + tasks = Path(directory) / 'ext' / 'tasks' + (tasks / 'x').mkdir(parents=True) + (tasks / '_index.json').write_text('{}') + (tasks / 'x' / 'history_item.json').write_text('{broken') + items, failures = load_items(Path(directory), 0) + self.assertEqual({}, items) + self.assertEqual(2, len(failures)) + + def test_id_mismatch_and_non_object_fail(self): + with tempfile.TemporaryDirectory() as directory: + tasks = Path(directory) / 'ext' / 'tasks' + for name, text in (('x', '{"id": "y"}'), ('z', '[1]')): + (tasks / name).mkdir(parents=True) + (tasks / name / 'history_item.json').write_text(text) + items, failures = load_items(Path(directory), 0) + self.assertEqual({}, items) + self.assertEqual(["x: id field is 'y'", 'z: history_item.json is not an object'], failures) + + def test_old_tasks_are_ignored(self): + with tempfile.TemporaryDirectory() as directory: + task = Path(directory) / 'ext' / 'tasks' / 'x' + task.mkdir(parents=True) + (task / 'history_item.json').write_text('{broken') + self.assertEqual(({}, []), load_items(Path(directory), 2 ** 40)) + + def test_provider_and_log_checks(self): + with tempfile.TemporaryDirectory() as directory: + provider = Path(directory) / 'p.jsonl' + good = ['barrier-released'] * 2 + ['completed'] * 6 + provider.write_text('\n'.join(json.dumps({'event': e}) for e in good)) + self.assertEqual([], check_provider(provider, 2)) + provider.write_text(json.dumps({'event': 'barrier-timeout'})) + self.assertTrue(check_provider(provider, 2)) + def provider_failures(events): + provider.write_text('\n'.join(json.dumps({'event': e}) for e in events)) + return check_provider(provider, 2) + self.assertIn('provider logged disconnected', provider_failures(good + ['disconnected'])) + self.assertIn('barrier released 1 times, expected 2', + provider_failures(['barrier-released'] + ['completed'] * 6)) + self.assertIn('5 completed streams, expected 6', + provider_failures(['barrier-released'] * 2 + ['completed'] * 5)) + self.assertIn('7 completed streams, expected 6', provider_failures(good + ['completed'])) + self.assertIn('barrier released 3 times, expected 2', provider_failures(good + ['barrier-released'])) + self.assertEqual(['provider logged barrier-timeout'], provider_failures(good + ['barrier-timeout'])) + provider.write_text('{"event": "completed"}\n{broken') + self.assertTrue(any('not valid JSON' in f for f in check_provider(provider, 2))) + provider.write_text('[1]\n' + '\n'.join(json.dumps({'event': e}) for e in good)) + self.assertEqual(["p.jsonl: provider log line is not an object: [1]"], check_provider(provider, 2)) + missing = Path(directory) / 'missing.log' + self.assertEqual([f'IDE log does not exist: {missing}'], check_logs([missing])) + log = Path(directory) / 'idea.log' + log.write_text('ok\nInvalid status transition interrupted -> delegated\n') + self.assertEqual(1, len(check_logs([log]))) + for phrase in ('invalid status transition', 'DeltaRejectedError', 'failed to persist', 'Cannot re-delegate'): + log.write_text(f'ok\n{phrase}\n') + self.assertEqual([f'idea.log:2: {phrase}'], check_logs([log]), phrase) + + +if __name__ == '__main__': + unittest.main() diff --git a/jetbrains_plugin/scripts/ui-harness/persist/test_solheim_proxy.py b/jetbrains_plugin/scripts/ui-harness/persist/test_solheim_proxy.py new file mode 100644 index 0000000..54e857a --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/persist/test_solheim_proxy.py @@ -0,0 +1,156 @@ +"""Local relay access checks; never contact the provider.""" +import http.client +import http.server +import threading +import urllib.error +import unittest +from unittest.mock import patch +import solheim_proxy as relay + + +class RelayAccessTest(unittest.TestCase): + def setUp(self): + relay.TOKEN = 'a' * 64 + self.server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), relay.Relay) + self.worker = threading.Thread(target=self.server.serve_forever) + self.worker.start() + + def tearDown(self): + self.server.shutdown() + self.server.server_close() + self.worker.join() + + def request(self, headers, body=None, method='GET'): + connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) + with patch.object(relay.UPSTREAM, 'open') as upstream: + connection.request(method, '/v1/models', body=body, headers=headers) + response = connection.getresponse() + response.read() + connection.close() + upstream.assert_not_called() + return response.status + + def test_local_health_is_authenticated_and_does_not_contact_upstream(self): + for token, expected in [('wrong', 401), (relay.TOKEN, 200)]: + connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) + with patch.object(relay.UPSTREAM, 'open') as upstream: + connection.request('GET', '/v1/health', headers={'Authorization': 'Bearer ' + token}) + response = connection.getresponse() + self.assertEqual(expected, response.status) + response.read() + upstream.assert_not_called() + connection.close() + + def test_missing_token(self): + self.assertEqual(401, self.request({})) + + def test_wrong_token(self): + self.assertEqual(401, self.request({'Authorization': 'Bearer wrong'})) + + def test_browser_origin(self): + self.assertEqual(403, self.request({'Authorization': 'Bearer ' + relay.TOKEN, + 'Origin': 'https://example.com'})) + + def test_untrusted_host(self): + self.assertEqual(403, self.request({'Authorization': 'Bearer ' + relay.TOKEN, + 'Host': 'example.com'})) + + def test_invalid_content_length(self): + for length in ['-1', 'not-a-number', '1.5']: + with self.subTest(length=length): + self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN, + 'Content-Length': length}, method='POST')) + + def test_excessive_content_length(self): + for length in [str(relay.MAX_BODY_BYTES + 1), '9' * 5000]: + with self.subTest(length=length[:20]): + self.assertEqual(413, self.request({'Authorization': 'Bearer ' + relay.TOKEN, + 'Content-Length': length}, method='POST')) + + def test_malformed_json(self): + for body in [b'{', b'\xff']: + with self.subTest(body=body): + self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN}, + body=body, method='POST')) + + def test_non_object_json(self): + for body in [b'[]', b'null', b'1']: + with self.subTest(body=body): + self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN}, + body=body, method='POST')) + + def test_wrong_message_shape(self): + self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN}, + body=b'{"messages":null}', method='POST')) + + def test_unsupported_transfer_encoding(self): + self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN, + 'Transfer-Encoding': 'chunked'}, method='POST')) + + def test_authorized_request_forwards_only_upstream_key(self): + relay.KEY = 'private-upstream-test-key' + response = unittest.mock.Mock(status=200, headers={'Content-Type': 'application/json'}) + response.read1.side_effect = [b'{"data":[]}', b''] + connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) + with patch.object(relay.UPSTREAM, 'open', return_value=response) as upstream, patch.object(relay, 'record'): + connection.request('GET', '/v1/models', headers={'Authorization': 'Bearer ' + relay.TOKEN}) + result = connection.getresponse() + self.assertEqual(200, result.status) + self.assertEqual(b'{"data":[]}', result.read()) + self.assertEqual('Bearer private-upstream-test-key', upstream.call_args.args[0].get_header('Authorization')) + connection.close() + + def test_authorized_post_forwards_json_unchanged(self): + relay.KEY = 'private-upstream-test-key' + body = b'{"model":"test-model","messages":[],"stream":true}' + response = unittest.mock.Mock(status=200, headers={'Content-Type': 'text/event-stream'}) + response.read1.side_effect = [b'data: [DONE]\n', b''] + connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) + with patch.object(relay.UPSTREAM, 'open', return_value=response) as upstream, patch.object(relay, 'record'): + connection.request('POST', '/v1/chat/completions', body=body, + headers={'Authorization': 'Bearer ' + relay.TOKEN}) + result = connection.getresponse() + self.assertEqual(200, result.status) + self.assertEqual(b'data: [DONE]\n', result.read()) + self.assertEqual(body, upstream.call_args.args[0].data) + self.assertEqual('POST', upstream.call_args.args[0].get_method()) + connection.close() + + +class RedirectTest(unittest.TestCase): + def test_redirects_never_reach_destination(self): + class RedirectServer(http.server.BaseHTTPRequestHandler): + paths = [] + + def log_message(self, *_): + pass + + def do_GET(self): + self.paths.append(self.path) + self.send_response(int(self.path[1:])) + self.send_header('Location', '/destination') + self.send_header('Content-Length', '0') + self.end_headers() + + server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), RedirectServer) + worker = threading.Thread(target=server.serve_forever) + worker.start() + try: + for status in [301, 302, 303, 307, 308]: + with self.subTest(status=status): + request = relay.urllib.request.Request( + 'http://127.0.0.1:' + str(server.server_port) + '/' + str(status), + headers={'Authorization': 'Bearer test-only-key'}) + with self.assertRaises(urllib.error.HTTPError) as caught: + relay.UPSTREAM.open(request, timeout=5) + self.assertEqual(status, caught.exception.code) + caught.exception.close() + self.assertEqual(['/301', '/302', '/303', '/307', '/308'], RedirectServer.paths) + finally: + server.shutdown() + server.server_close() + worker.join() + + +if __name__ == '__main__': + unittest.main() diff --git a/jetbrains_plugin/scripts/ui-harness/persist/test_streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/persist/test_streaming_fixture.py new file mode 100644 index 0000000..72f5a85 --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/persist/test_streaming_fixture.py @@ -0,0 +1,188 @@ +import http.client +import json +import tempfile +import threading +import unittest +from unittest.mock import patch +from pathlib import Path + +from streaming_fixture import history_role, make_server + + +class StreamingFixtureTest(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.log = Path(self.directory.name) / 'provider.jsonl' + self.server = make_server(0, self.log, chunks=3, interval=0.03) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + self.connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port, timeout=5) + + def tearDown(self): + self.connection.close() + self.server.shutdown() + self.server.server_close() + self.thread.join() + self.directory.cleanup() + + def post(self, body, token='focus-fixture-local-only'): + self.connection.request('POST', '/v1/chat/completions', json.dumps(body), + {'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json'}) + return self.connection.getresponse() + + def test_two_incremental_streams_finish_with_native_completion(self): + request = {'stream': True, 'messages': [], 'tools': [ + {'type': 'function', 'function': {'name': 'attempt_completion'}}]} + for turn in (1, 2): + paused = threading.Event() + release = threading.Event() + + def pause_stream(_interval): + paused.set() + if not release.wait(5): + raise TimeoutError('Test did not release the fixture stream') + + # Hold the server after its first content chunk. Receiving both the + # role and content before release proves incremental delivery without + # depending on sleep durations or the host's clock resolution. + with patch('streaming_fixture.time.sleep', side_effect=pause_stream): + try: + response = self.post(request) + self.assertEqual(200, response.status) + self.assertEqual('text/event-stream', response.getheader('Content-Type')) + first = json.loads(response.readline().decode()[6:]) + self.assertEqual('assistant', first['choices'][0]['delta']['role']) + self.assertEqual(b'\n', response.readline()) + update = json.loads(response.readline().decode()[6:]) + self.assertEqual(f'History update 1 for turn {turn}. ', + update['choices'][0]['delta']['content']) + self.assertTrue(paused.wait(5), 'Server did not pause during the stream') + finally: + # Also unblock the request handler if any assertion fails. + release.set() + events = [update] + [json.loads(line[6:]) for line in response.read().decode().splitlines() + if line.startswith('data: {')] + updates = [e for e in events if e['choices'][0]['delta'].get('content')] + self.assertEqual(3, len(updates)) + tool = events[-2]['choices'][0]['delta']['tool_calls'][0] + self.assertEqual('attempt_completion', tool['function']['name']) + self.assertIn(str(turn), json.loads(tool['function']['arguments'])['result']) + self.assertEqual('tool_calls', events[-1]['choices'][0]['finish_reason']) + records = [json.loads(line) for line in self.log.read_text().splitlines()] + self.assertEqual(2, len([e for e in records if e['event'] == 'completed'])) + + def test_rejects_non_streaming_and_missing_completion_tool(self): + for request in ({'stream': False}, {'stream': True, 'tools': []}): + response = self.post(request) + self.assertEqual(400, response.status) + response.read() + self.assertEqual(0, self.server.requests) + + def test_rejects_invalid_local_token(self): + response = self.post({'stream': True}, token='wrong') + self.assertEqual(401, response.status) + response.read() + self.assertEqual(0, self.server.requests) + + def test_health_and_model_discovery(self): + self.connection.request('GET', '/health') + response = self.connection.getresponse() + self.assertEqual(b'ready', response.read()) + self.connection.request('GET', '/v1/models') + response = self.connection.getresponse() + self.assertEqual('focus-fixture', json.loads(response.read())['data'][0]['id']) + + +class HistoryRaceScenarioTest(unittest.TestCase): + def user(self, text): + return {'role': 'user', 'content': [{'type': 'text', 'text': text}]} + + def test_roles_follow_the_first_user_message(self): + self.assertEqual(('delegate', 'A'), history_role([self.user('HISTORY_RACE_PARENT A go')])) + self.assertEqual(('child', 'B'), history_role([self.user('HISTORY_RACE_CHILD B go')])) + self.assertEqual(('plain', ''), history_role([self.user('hello')])) + + def test_parent_resumes_after_new_task_but_ignores_the_system_prompt(self): + system = {'role': 'system', 'content': 'tools: new_task, attempt_completion'} + first = self.user('HISTORY_RACE_PARENT A go') + self.assertEqual(('delegate', 'A'), history_role([system, first])) + called = {'role': 'assistant', 'tool_calls': [{'function': {'name': 'new_task'}}]} + self.assertEqual(('resume', 'A'), history_role([system, first, called])) + + def test_two_parents_delegate_together_and_children_complete(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + log = Path(directory.name) / 'provider.jsonl' + server = make_server(0, log, chunks=2, interval=0.01, scenario='history-race', barrier=2) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + tools = [{'type': 'function', 'function': {'name': 'attempt_completion'}}] + results = {} + + def call(tag, text): + connection = http.client.HTTPConnection('127.0.0.1', server.server_port, timeout=10) + body = {'stream': True, 'tools': tools, 'messages': [self.user(text)]} + connection.request('POST', '/v1/chat/completions', json.dumps(body), + {'Authorization': 'Bearer focus-fixture-local-only'}) + data = connection.getresponse().read().decode() + results[tag] = [json.loads(line[6:]) for line in data.splitlines() if line.startswith('data: {')] + connection.close() + + def events(): + return [json.loads(line)['event'] for line in log.read_text().splitlines()] if log.exists() else [] + + parents = {tag: threading.Thread(target=call, args=(tag, f'HISTORY_RACE_PARENT {tag}')) for tag in 'AB'} + # Parent A alone must stay blocked: no new_task and no barrier-released. + parents['A'].start() + parents['A'].join(0.5) + self.assertTrue(parents['A'].is_alive()) + self.assertNotIn('A', results) + self.assertNotIn('barrier-released', events()) + parents['B'].start() + for parent in parents.values(): + parent.join(15) + self.assertFalse(parent.is_alive()) + for tag in 'AB': + call_delta = results[tag][-2]['choices'][0]['delta']['tool_calls'][0]['function'] + self.assertEqual('new_task', call_delta['name']) + arguments = json.loads(call_delta['arguments']) + self.assertEqual(f'HISTORY_RACE_CHILD {tag} reply with attempt_completion.', arguments['message']) + self.assertIsNone(arguments['todos']) + call('A', 'HISTORY_RACE_CHILD A go') + done = results['A'][-2]['choices'][0]['delta']['tool_calls'][0]['function']['name'] + self.assertEqual('attempt_completion', done) + events = [json.loads(line)['event'] for line in log.read_text().splitlines()] + self.assertEqual(2, events.count('barrier-released')) + self.assertNotIn('barrier-timeout', events) + counts = [json.loads(line)['chunks'] for line in log.read_text().splitlines() + if json.loads(line)['event'] == 'completed'] + self.assertEqual([2, 3, 3], sorted(counts)) + + def test_lone_parent_times_out_instead_of_passing(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + log = Path(directory.name) / 'provider.jsonl' + server = make_server(0, log, chunks=1, interval=0.01, scenario='history-race', + barrier=2, barrier_timeout=0.3) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + connection = http.client.HTTPConnection('127.0.0.1', server.server_port, timeout=10) + body = {'stream': True, 'tools': [{'type': 'function', 'function': {'name': 'attempt_completion'}}], + 'messages': [self.user('HISTORY_RACE_PARENT A go')]} + connection.request('POST', '/v1/chat/completions', json.dumps(body), + {'Authorization': 'Bearer focus-fixture-local-only'}) + connection.getresponse().read() + connection.close() + events = [json.loads(line)['event'] for line in log.read_text().splitlines()] + self.assertIn('barrier-timeout', events) + self.assertNotIn('barrier-released', events) + + +if __name__ == '__main__': + unittest.main() diff --git a/jetbrains_plugin/scripts/ui-harness/re-delegate/check_history_store.py b/jetbrains_plugin/scripts/ui-harness/re-delegate/check_history_store.py new file mode 100644 index 0000000..b76de99 --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/re-delegate/check_history_store.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Check the shared Zoo Code task history after a concurrent two-IDE run. + +Issue #6: several IDE processes write one global history store. A safe store +keeps one valid history_item.json per task, with no shared index file and with +parent and child links that stay inside one IDE's task tree. +""" +import argparse +import json +import re +import sys +from pathlib import Path + +PLACEHOLDER = re.compile(r'^\s*(Task|Work) #\d+') +BAD_LOG = re.compile(r'invalid status transition|DeltaRejectedError|failed to persist|Cannot re-delegate', re.I) +MARKER = re.compile(r'HISTORY_RACE_(PARENT|CHILD) (\w+)') + + +def load_items(storage, since): + """Return ({task id: item}, failures) for tasks written at or after `since`.""" + items, failures = {}, [] + for index in storage.glob('**/tasks/_index.json'): + if index.stat().st_mtime >= since: + failures.append(f'shared index file was written: {index}') + for path in sorted(storage.glob('**/tasks/*/history_item.json')): + if path.stat().st_mtime < since: + continue + try: + item = json.loads(path.read_text(encoding='utf-8')) + except (OSError, ValueError) as error: + failures.append(f'{path.parent.name}: history_item.json is not valid JSON ({error})') + continue + if not isinstance(item, dict): + failures.append(f'{path.parent.name}: history_item.json is not an object') + continue + if item.get('id') != path.parent.name: + failures.append(f'{path.parent.name}: id field is {item.get("id")!r}') + continue + items[item['id']] = item + return items, failures + + +def check_items(items, instances): + failures = [] + tasks = {} + for task_id, item in items.items(): + text = item.get('task') or '' + if PLACEHOLDER.match(text) or not text.strip(): + failures.append(f'{task_id}: placeholder or empty title {text!r}') + continue + match = MARKER.search(text) + if not match: + failures.append(f'{task_id}: title does not carry a run marker: {text!r}') + continue + if not item.get('size'): + failures.append(f'{task_id}: size is {item.get("size")!r} after real work') + tasks[task_id] = (match.group(1), match.group(2), item) + parents = {i: t for i, t in tasks.items() if t[0] == 'PARENT'} + children = {i: t for i, t in tasks.items() if t[0] == 'CHILD'} + if len(parents) != instances or len(children) != instances: + failures.append(f'expected {instances} parents and {instances} children, ' + f'found {len(parents)} and {len(children)}') + for parent_id, (_, tag, parent) in parents.items(): + own = [c for c, t in children.items() if t[1] == tag] + if len(own) != 1: + failures.append(f'{parent_id}: expected one child for tag {tag}, found {len(own)}') + continue + child_id = own[0] + child = children[child_id][2] + if parent.get('childIds') != [child_id]: + failures.append(f'{parent_id}: childIds {parent.get("childIds")!r}, expected {[child_id]!r}') + if child.get('parentTaskId') != parent_id: + failures.append(f'{child_id}: parentTaskId {child.get("parentTaskId")!r}, expected {parent_id!r}') + if child.get('status') != 'completed': + failures.append(f'{child_id}: child status {child.get("status")!r}, expected completed') + if parent.get('status') == 'delegated' or parent.get('awaitingChildId') or parent.get('delegatedToId'): + failures.append(f'{parent_id}: parent still waits on a child (status {parent.get("status")!r})') + if parent.get('workspace') != child.get('workspace'): + failures.append(f'{parent_id}: parent and child workspaces differ') + workspaces = {t[2].get('workspace') for t in parents.values()} + if instances > 1 and len(workspaces) != instances: + failures.append(f'expected {instances} distinct workspaces, found {sorted(map(str, workspaces))}') + return failures + + +def check_provider(path, instances): + events, failures = [], [] + for number, line in enumerate(path.read_text(encoding='utf-8').splitlines(), 1): + if not line.strip(): + continue + try: + events.append(json.loads(line)) + except ValueError as error: + failures.append(f'{path.name}:{number}: provider log line is not valid JSON ({error})') + for event in events: + if not isinstance(event, dict): + failures.append(f'{path.name}: provider log line is not an object: {event!r}') + names = [e.get('event') for e in events if isinstance(e, dict)] + if names.count('barrier-released') != instances: + failures.append(f'barrier released {names.count("barrier-released")} times, expected {instances}') + for bad in ('barrier-timeout', 'disconnected'): + if bad in names: + failures.append(f'provider logged {bad}') + if names.count('completed') != 3 * instances: + failures.append(f'{names.count("completed")} completed streams, expected {3 * instances}') + return failures + + +def check_logs(paths): + failures = [] + for path in paths: + if not path.exists(): + failures.append(f'IDE log does not exist: {path}') + continue + for number, line in enumerate(path.read_text(encoding='utf-8', errors='replace').splitlines(), 1): + if BAD_LOG.search(line): + failures.append(f'{path.name}:{number}: {line.strip()[:200]}') + return failures + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--storage', type=Path, required=True) + parser.add_argument('--since', type=float, required=True, help='epoch seconds when the run started') + parser.add_argument('--provider-log', type=Path, required=True) + parser.add_argument('--ide-log', type=Path, action='append', required=True) + parser.add_argument('--instances', type=int, default=2) + parser.add_argument('--report', type=Path) + args = parser.parse_args() + items, failures = load_items(args.storage, args.since) + failures += check_items(items, args.instances) + failures += check_provider(args.provider_log, args.instances) + failures += check_logs(args.ide_log) + report = {'tasks': len(items), 'failures': failures} + text = json.dumps(report, indent=2) + if args.report: + args.report.write_text(text, encoding='utf-8') + print(text) + return 1 if failures else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/jetbrains_plugin/scripts/ui-harness/re-delegate/solheim_proxy.py b/jetbrains_plugin/scripts/ui-harness/re-delegate/solheim_proxy.py new file mode 100644 index 0000000..669102d --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/re-delegate/solheim_proxy.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Loopback authentication relay for manual Solheim extension focus smoke tests. + +Real credentials stay in SOLHEIM_KEY_FILE (mode 0600), outside IDE settings/logs. +The extension uses a per-run relay token and http://127.0.0.1:19487/v1. Logs contain only +request metadata, upstream status, stream byte counts, and tool names. +""" +import http.server +import hmac +import json +import os +import time +import urllib.error +import urllib.request +from pathlib import Path + +KEY = None +TOKEN = None +BASE = 'https://api.solheim.ai/v1' +MAX_BODY_BYTES = 8 * 1024 * 1024 +LOG = Path(os.environ.get('SOLHEIM_PROXY_LOG', '/tmp/zoo-solheim-proxy.jsonl')) + + +def record(**fields): + with LOG.open('a') as output: + output.write(json.dumps({'time': time.time(), **fields}) + '\n') + + +class RefuseRedirects(urllib.request.HTTPRedirectHandler): + def redirect_request(self, request, response, code, message, headers, new_url): + # Never send provider credentials to a redirected URL. + return None + + +UPSTREAM = urllib.request.build_opener(RefuseRedirects()) + + +class Relay(http.server.BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def log_message(self, *_): + pass + + def do_GET(self): + self.relay() + + def do_POST(self): + self.relay() + + def relay(self): + expected_host = '127.0.0.1:' + str(self.server.server_port) + if self.headers.get('Host') != expected_host or self.headers.get('Origin') is not None: + self.send_error(403) + return + supplied = self.headers.get('Authorization', '') + if not TOKEN or not hmac.compare_digest(supplied, 'Bearer ' + TOKEN): + self.send_error(401) + return + if not self.path.startswith('/v1/'): + self.send_error(404) + return + if self.command == 'GET' and self.path == '/v1/health': + # Local readiness only; never probe the paid upstream in a startup loop. + self.send_response(200) + self.send_header('Content-Length', '5') + self.end_headers() + self.wfile.write(b'ready') + return + lengths = self.headers.get_all('Content-Length', []) + length_text = lengths[0] if lengths else '0' + if (len(lengths) > 1 or self.headers.get('Transfer-Encoding') is not None + or not length_text.isascii() or not length_text.isdecimal()): + self.send_error(400, 'Invalid request framing') + return + # Check before int conversion, including absurdly long decimal strings. + if len(length_text.lstrip('0')) > 8 or int(length_text.lstrip('0') or '0') > MAX_BODY_BYTES: + self.send_error(413, 'Request body too large') + return + length = int(length_text.lstrip('0') or '0') + body = self.rfile.read(length) + try: + parsed = json.loads(body) if body else {} + if len(body) != length or not isinstance(parsed, dict): + raise ValueError('Expected a JSON object') + if not isinstance(parsed.get('messages', []), list): + raise ValueError('Expected a message list') + except (ValueError, UnicodeError): + self.send_error(400, 'Invalid JSON request body') + return + record(event='request', path=self.path, model=parsed.get('model'), + messages=len(parsed.get('messages', [])), stream=parsed.get('stream', False)) + request = urllib.request.Request(BASE + self.path[3:], data=body or None, + method=self.command, + headers={'Authorization': 'Bearer ' + KEY, + 'Content-Type': 'application/json'}) + try: + response = UPSTREAM.open(request, timeout=180) + except urllib.error.HTTPError as error: + response = error + except Exception as error: + record(event='transport-error', error=type(error).__name__) + self.send_error(502) + return + self.send_response(response.status) + self.send_header('Content-Type', response.headers.get('Content-Type', 'application/json')) + self.send_header('Connection', 'close') + self.end_headers() + count = 0 + buffer = b'' + tools = set() + try: + while True: + chunk = response.read1(4096) + if not chunk: + break + count += len(chunk) + self.wfile.write(chunk) + self.wfile.flush() + buffer += chunk + while b'\n' in buffer: + line, buffer = buffer.split(b'\n', 1) + if line.startswith(b'data: ') and line != b'data: [DONE]': + try: + event = json.loads(line[6:]) + for choice in event.get('choices', []): + for tool in choice.get('delta', {}).get('tool_calls', []): + name = tool.get('function', {}).get('name') + if name: + tools.add(name) + except (ValueError, TypeError): + pass + except (BrokenPipeError, ConnectionResetError): + record(event='client-disconnected') + finally: + response.close() + self.close_connection = True + record(event='response', status=response.status, bytes=count, tools=sorted(tools)) + + +if __name__ == '__main__': + KEY = Path(os.environ['SOLHEIM_KEY_FILE']).read_text().strip() + TOKEN = os.environ['SOLHEIM_RELAY_TOKEN'] + if len(TOKEN) < 32: + raise SystemExit('Use a random relay token of at least 32 characters') + http.server.ThreadingHTTPServer(('127.0.0.1', 19487), Relay).serve_forever() diff --git a/jetbrains_plugin/scripts/ui-harness/re-delegate/streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/re-delegate/streaming_fixture.py new file mode 100644 index 0000000..07d7af8 --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/re-delegate/streaming_fixture.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Deterministic loopback OpenAI fixture for headful chat-focus CI tests. + +Only the provider is simulated. The extension, React history, JCEF, and native +keyboard delivery remain real. No commands or file edits are requested. +""" +import argparse +import http.server +import json +import re +import threading +import time +from pathlib import Path + + +def _text(content): + if isinstance(content, list): + return ''.join(part.get('text', '') for part in content if isinstance(part, dict)) + return content if isinstance(content, str) else '' + + +def history_role(messages): + """Classify a request in the history-race scenario. + + The first user message names the task: a parent carries HISTORY_RACE_PARENT + and a child carries HISTORY_RACE_CHILD . A parent that already holds a + new_task call in a non-system message has delegated, so it resumes. + """ + users = [m for m in messages if m.get('role') == 'user'] + first = _text(users[0].get('content')) if users else '' + child = re.search(r'HISTORY_RACE_CHILD (\w+)', first) + if child: + return 'child', child.group(1) + parent = re.search(r'HISTORY_RACE_PARENT (\w+)', first) + if not parent: + return 'plain', '' + body = json.dumps([m for m in messages if m.get('role') != 'system']) + return ('resume' if 'new_task' in body else 'delegate'), parent.group(1) + + +class Fixture(http.server.BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_GET(self): + if self.path == '/health': + self.send_response(200) + self.end_headers() + self.wfile.write(b'ready') + elif self.path == '/v1/models': + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps({'object': 'list', 'data': [ + {'id': 'focus-fixture', 'object': 'model', 'owned_by': 'fixture'} + ]}).encode()) + else: + self.send_error(404) + + def do_POST(self): + if self.path != '/v1/chat/completions': + self.send_error(404) + return + if self.headers.get('Authorization') != 'Bearer focus-fixture-local-only': + self.send_error(401) + return + try: + length = int(self.headers.get('Content-Length', '0')) + if not 0 < length <= 8 * 1024 * 1024: + raise ValueError('body size') + request = json.loads(self.rfile.read(length)) + if request.get('stream') is not True: + raise ValueError('streaming required') + tools = [t.get('function', {}).get('name') for t in request.get('tools', [])] + if 'attempt_completion' not in tools: + raise ValueError('native attempt_completion tool required') + except (ValueError, TypeError, AttributeError): + self.send_error(400, 'Expected streaming request with native completion tool') + return + with self.server.counter_lock: + self.server.requests += 1 + turn = self.server.requests + role, tag = 'plain', '' + if self.server.scenario == 'history-race': + role, tag = history_role(request.get('messages', [])) + self.server.record(event='request', turn=turn, tools=tools, role=role, tag=tag) + self.send_response(200) + self.send_header('Content-Type', 'text/event-stream') + self.send_header('Cache-Control', 'no-cache') + self.end_headers() + + def emit(delta, finish=None): + chunk = {'id': f'focus-{turn}', 'object': 'chat.completion.chunk', + 'created': int(time.time()), 'model': 'focus-fixture', + 'choices': [{'index': 0, 'delta': delta, 'finish_reason': finish}]} + self.wfile.write(('data: ' + json.dumps(chunk) + '\n\n').encode()) + self.wfile.flush() + + try: + emit({'role': 'assistant', 'content': ''}) + chunks = self.server.chunks + if role == 'delegate': + # Hold every parent until all IDEs have submitted, so the delegations + # and the child streams that follow overlap in time. + try: + self.server.barrier.wait() + except threading.BrokenBarrierError: + self.server.record(event='barrier-timeout', turn=turn, tag=tag) + return + self.server.record(event='barrier-released', turn=turn, tag=tag) + if role in ('delegate', 'resume'): + chunks = 3 + for index in range(chunks): + emit({'content': f'History update {index + 1} for turn {turn}. '}) + time.sleep(self.server.interval) + if role == 'delegate': + call = ('new_task', {'mode': 'code', 'todos': None, + 'message': f'HISTORY_RACE_CHILD {tag} reply with attempt_completion.'}) + else: + call = ('attempt_completion', {'result': f'Fixture turn {turn} complete.'}) + emit({'tool_calls': [{'index': 0, 'id': f'call-{turn}', + 'type': 'function', 'function': { + 'name': call[0], 'arguments': json.dumps(call[1])}}]}) + emit({}, 'tool_calls') + self.wfile.write(b'data: [DONE]\n\n') + self.wfile.flush() + self.server.record(event='completed', turn=turn, chunks=chunks) + except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError): + self.server.record(event='disconnected', turn=turn) + + +def make_server(port=19487, log=None, chunks=100, interval=0.25, scenario=None, barrier=2, barrier_timeout=300): + server = http.server.ThreadingHTTPServer(('127.0.0.1', port), Fixture) + server.requests = 0 + server.counter_lock = threading.Lock() + server.chunks, server.interval = chunks, interval + server.scenario = scenario + server.barrier = threading.Barrier(barrier, timeout=barrier_timeout) + + def record(**fields): + if log: + with server.counter_lock, Path(log).open('a', encoding='utf-8') as output: + output.write(json.dumps({'time': time.time(), **fields}) + '\n') + server.record = record + return server + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--port', type=int, default=19487) + parser.add_argument('--log', type=Path, required=True) + parser.add_argument('--scenario', choices=['history-race']) + parser.add_argument('--chunks', type=int, default=100) + parser.add_argument('--interval', type=float, default=0.25) + parser.add_argument('--barrier', type=int, default=2) + args = parser.parse_args() + make_server(args.port, args.log, args.chunks, args.interval, args.scenario, args.barrier).serve_forever() diff --git a/jetbrains_plugin/scripts/ui-harness/re-delegate/test_check_history_store.py b/jetbrains_plugin/scripts/ui-harness/re-delegate/test_check_history_store.py new file mode 100644 index 0000000..b095026 --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/re-delegate/test_check_history_store.py @@ -0,0 +1,235 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from check_history_store import check_items, check_logs, check_provider, load_items + + +def item(task_id, text, **extra): + return {'id': task_id, 'task': text, 'size': 10, 'workspace': 'w', **extra} + + +def good_pair(tag, workspace): + return { + 'p' + tag: item('p' + tag, f'HISTORY_RACE_PARENT {tag} go', childIds=['c' + tag], + status='active', workspace=workspace), + 'c' + tag: item('c' + tag, f'HISTORY_RACE_CHILD {tag} go', parentTaskId='p' + tag, + status='completed', workspace=workspace), + } + + +# Real items from Windows CI run 43562237505 (commit 5a494cf, IDEA 2026.2.1, extension 3.96.0). +# Only the temporary run folder name in `workspace` was shortened. +REAL_ITEMS = json.loads(r'''[ + { + "id": "01a1d898-c7ab-70da-936e-6467341e0125", + "parentTaskId": "01a1d898-bf28-71a1-92d9-df474bf72871", + "number": 1, + "ts": 1794635304779, + "task": "HISTORY_RACE_CHILD A reply with attempt_completion.", + "tokensIn": 0, + "tokensOut": 0, + "totalCost": 0, + "size": 10621, + "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-A", + "mode": "code", + "apiConfigName": "focus-fixture", + "status": "completed", + "completionResultSummary": "Fixture turn 4 complete." + }, + { + "id": "01a1d898-c663-72e9-b6dd-1caba4bda93a", + "parentTaskId": "01a1d898-4435-75fd-afc2-6ca3aa4c89a4", + "number": 1, + "ts": 1794635304331, + "task": "HISTORY_RACE_CHILD B reply with attempt_completion.", + "tokensIn": 0, + "tokensOut": 0, + "totalCost": 0, + "size": 10623, + "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-B", + "mode": "code", + "apiConfigName": "focus-fixture", + "status": "completed", + "completionResultSummary": "Fixture turn 3 complete." + }, + { + "id": "01a1d898-bf28-71a1-92d9-df474bf72871", + "number": 1, + "ts": 1794635306508, + "task": "HISTORY_RACE_PARENT A delegate one subtask, then finish.", + "tokensIn": 0, + "tokensOut": 0, + "totalCost": 0, + "size": 7694, + "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-A", + "mode": "code", + "apiConfigName": "focus-fixture", + "status": "active", + "childIds": [ + "01a1d898-c7ab-70da-936e-6467341e0125" + ], + "completedByChildId": "01a1d898-c7ab-70da-936e-6467341e0125", + "completionResultSummary": "Fixture turn 4 complete." + }, + { + "id": "01a1d898-4435-75fd-afc2-6ca3aa4c89a4", + "number": 1, + "ts": 1794635306029, + "task": "HISTORY_RACE_PARENT B delegate one subtask, then finish.", + "tokensIn": 0, + "tokensOut": 0, + "totalCost": 0, + "size": 7698, + "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-B", + "mode": "code", + "apiConfigName": "focus-fixture", + "status": "active", + "childIds": [ + "01a1d898-c663-72e9-b6dd-1caba4bda93a" + ], + "completedByChildId": "01a1d898-c663-72e9-b6dd-1caba4bda93a", + "completionResultSummary": "Fixture turn 3 complete." + } +]''') + + +class CheckItemsTest(unittest.TestCase): + def test_real_extension_items_pass(self): + items = {i['id']: i for i in REAL_ITEMS} + self.assertEqual(4, len(items)) + self.assertEqual([], check_items(items, 2)) + + + def two(self): + return {**good_pair('A', 'wa'), **good_pair('B', 'wb')} + + def test_clean_store_passes(self): + self.assertEqual([], check_items(self.two(), 2)) + + def test_placeholder_title_and_zero_size_fail(self): + items = self.two() + items['pA']['task'] = 'Task #1' + items['cB']['size'] = 0 + failures = check_items(items, 2) + self.assertTrue(any('placeholder' in f for f in failures)) + self.assertTrue(any('size is 0' in f for f in failures)) + + def test_cross_linked_children_fail(self): + items = self.two() + items['pA']['childIds'] = ['cB'] + items['cA']['parentTaskId'] = 'pB' + failures = check_items(items, 2) + self.assertTrue(any('childIds' in f for f in failures)) + self.assertTrue(any('parentTaskId' in f for f in failures)) + + def test_stuck_delegation_and_open_child_fail(self): + items = self.two() + items['pA'].update(status='delegated', awaitingChildId='cA') + items['cA']['status'] = 'interrupted' + failures = check_items(items, 2) + self.assertTrue(any('still waits' in f for f in failures)) + self.assertTrue(any('child status' in f for f in failures)) + + def fails(self, items, message): + self.assertIn(message, check_items(items, 2)) + + def test_each_bad_item_branch_reports_an_exact_message(self): + items = self.two() + items['pA']['task'] = 'Work #2' + self.fails(items, "pA: placeholder or empty title 'Work #2'") + items = self.two() + items['pA']['task'] = ' ' + self.fails(items, "pA: placeholder or empty title ' '") + items = self.two() + items['pA']['task'] = 'no marker' + self.fails(items, "pA: title does not carry a run marker: 'no marker'") + items = self.two() + items['pA']['delegatedToId'] = 'cA' + self.fails(items, 'pA: parent still waits on a child (status \'active\')') + items = self.two() + items['pA']['awaitingChildId'] = 'cA' + self.fails(items, 'pA: parent still waits on a child (status \'active\')') + items = self.two() + items['cA']['workspace'] = 'other' + self.fails(items, 'pA: parent and child workspaces differ') + + def test_shared_workspace_and_extra_child_fail(self): + items = {**good_pair('A', 'w'), **good_pair('B', 'w')} + self.assertTrue(any('distinct workspaces' in f for f in check_items(items, 2))) + items = self.two() + items.update(good_pair('A', 'wa')) + items['cA2'] = item('cA2', 'HISTORY_RACE_CHILD A again', parentTaskId='pA', + status='completed', workspace='wa') + self.fails(items, 'pA: expected one child for tag A, found 2') + + def test_missing_task_fails(self): + items = self.two() + del items['cB'] + self.assertTrue(any('expected 2 parents' in f for f in check_items(items, 2))) + + +class StoreAndLogTest(unittest.TestCase): + def test_index_file_and_bad_json_fail(self): + with tempfile.TemporaryDirectory() as directory: + tasks = Path(directory) / 'ext' / 'tasks' + (tasks / 'x').mkdir(parents=True) + (tasks / '_index.json').write_text('{}') + (tasks / 'x' / 'history_item.json').write_text('{broken') + items, failures = load_items(Path(directory), 0) + self.assertEqual({}, items) + self.assertEqual(2, len(failures)) + + def test_id_mismatch_and_non_object_fail(self): + with tempfile.TemporaryDirectory() as directory: + tasks = Path(directory) / 'ext' / 'tasks' + for name, text in (('x', '{"id": "y"}'), ('z', '[1]')): + (tasks / name).mkdir(parents=True) + (tasks / name / 'history_item.json').write_text(text) + items, failures = load_items(Path(directory), 0) + self.assertEqual({}, items) + self.assertEqual(["x: id field is 'y'", 'z: history_item.json is not an object'], failures) + + def test_old_tasks_are_ignored(self): + with tempfile.TemporaryDirectory() as directory: + task = Path(directory) / 'ext' / 'tasks' / 'x' + task.mkdir(parents=True) + (task / 'history_item.json').write_text('{broken') + self.assertEqual(({}, []), load_items(Path(directory), 2 ** 40)) + + def test_provider_and_log_checks(self): + with tempfile.TemporaryDirectory() as directory: + provider = Path(directory) / 'p.jsonl' + good = ['barrier-released'] * 2 + ['completed'] * 6 + provider.write_text('\n'.join(json.dumps({'event': e}) for e in good)) + self.assertEqual([], check_provider(provider, 2)) + provider.write_text(json.dumps({'event': 'barrier-timeout'})) + self.assertTrue(check_provider(provider, 2)) + def provider_failures(events): + provider.write_text('\n'.join(json.dumps({'event': e}) for e in events)) + return check_provider(provider, 2) + self.assertIn('provider logged disconnected', provider_failures(good + ['disconnected'])) + self.assertIn('barrier released 1 times, expected 2', + provider_failures(['barrier-released'] + ['completed'] * 6)) + self.assertIn('5 completed streams, expected 6', + provider_failures(['barrier-released'] * 2 + ['completed'] * 5)) + self.assertIn('7 completed streams, expected 6', provider_failures(good + ['completed'])) + self.assertIn('barrier released 3 times, expected 2', provider_failures(good + ['barrier-released'])) + self.assertEqual(['provider logged barrier-timeout'], provider_failures(good + ['barrier-timeout'])) + provider.write_text('{"event": "completed"}\n{broken') + self.assertTrue(any('not valid JSON' in f for f in check_provider(provider, 2))) + provider.write_text('[1]\n' + '\n'.join(json.dumps({'event': e}) for e in good)) + self.assertEqual(["p.jsonl: provider log line is not an object: [1]"], check_provider(provider, 2)) + missing = Path(directory) / 'missing.log' + self.assertEqual([f'IDE log does not exist: {missing}'], check_logs([missing])) + log = Path(directory) / 'idea.log' + log.write_text('ok\nInvalid status transition interrupted -> delegated\n') + self.assertEqual(1, len(check_logs([log]))) + for phrase in ('invalid status transition', 'DeltaRejectedError', 'failed to persist', 'Cannot re-delegate'): + log.write_text(f'ok\n{phrase}\n') + self.assertEqual([f'idea.log:2: {phrase}'], check_logs([log]), phrase) + + +if __name__ == '__main__': + unittest.main() diff --git a/jetbrains_plugin/scripts/ui-harness/re-delegate/test_solheim_proxy.py b/jetbrains_plugin/scripts/ui-harness/re-delegate/test_solheim_proxy.py new file mode 100644 index 0000000..54e857a --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/re-delegate/test_solheim_proxy.py @@ -0,0 +1,156 @@ +"""Local relay access checks; never contact the provider.""" +import http.client +import http.server +import threading +import urllib.error +import unittest +from unittest.mock import patch +import solheim_proxy as relay + + +class RelayAccessTest(unittest.TestCase): + def setUp(self): + relay.TOKEN = 'a' * 64 + self.server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), relay.Relay) + self.worker = threading.Thread(target=self.server.serve_forever) + self.worker.start() + + def tearDown(self): + self.server.shutdown() + self.server.server_close() + self.worker.join() + + def request(self, headers, body=None, method='GET'): + connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) + with patch.object(relay.UPSTREAM, 'open') as upstream: + connection.request(method, '/v1/models', body=body, headers=headers) + response = connection.getresponse() + response.read() + connection.close() + upstream.assert_not_called() + return response.status + + def test_local_health_is_authenticated_and_does_not_contact_upstream(self): + for token, expected in [('wrong', 401), (relay.TOKEN, 200)]: + connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) + with patch.object(relay.UPSTREAM, 'open') as upstream: + connection.request('GET', '/v1/health', headers={'Authorization': 'Bearer ' + token}) + response = connection.getresponse() + self.assertEqual(expected, response.status) + response.read() + upstream.assert_not_called() + connection.close() + + def test_missing_token(self): + self.assertEqual(401, self.request({})) + + def test_wrong_token(self): + self.assertEqual(401, self.request({'Authorization': 'Bearer wrong'})) + + def test_browser_origin(self): + self.assertEqual(403, self.request({'Authorization': 'Bearer ' + relay.TOKEN, + 'Origin': 'https://example.com'})) + + def test_untrusted_host(self): + self.assertEqual(403, self.request({'Authorization': 'Bearer ' + relay.TOKEN, + 'Host': 'example.com'})) + + def test_invalid_content_length(self): + for length in ['-1', 'not-a-number', '1.5']: + with self.subTest(length=length): + self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN, + 'Content-Length': length}, method='POST')) + + def test_excessive_content_length(self): + for length in [str(relay.MAX_BODY_BYTES + 1), '9' * 5000]: + with self.subTest(length=length[:20]): + self.assertEqual(413, self.request({'Authorization': 'Bearer ' + relay.TOKEN, + 'Content-Length': length}, method='POST')) + + def test_malformed_json(self): + for body in [b'{', b'\xff']: + with self.subTest(body=body): + self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN}, + body=body, method='POST')) + + def test_non_object_json(self): + for body in [b'[]', b'null', b'1']: + with self.subTest(body=body): + self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN}, + body=body, method='POST')) + + def test_wrong_message_shape(self): + self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN}, + body=b'{"messages":null}', method='POST')) + + def test_unsupported_transfer_encoding(self): + self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN, + 'Transfer-Encoding': 'chunked'}, method='POST')) + + def test_authorized_request_forwards_only_upstream_key(self): + relay.KEY = 'private-upstream-test-key' + response = unittest.mock.Mock(status=200, headers={'Content-Type': 'application/json'}) + response.read1.side_effect = [b'{"data":[]}', b''] + connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) + with patch.object(relay.UPSTREAM, 'open', return_value=response) as upstream, patch.object(relay, 'record'): + connection.request('GET', '/v1/models', headers={'Authorization': 'Bearer ' + relay.TOKEN}) + result = connection.getresponse() + self.assertEqual(200, result.status) + self.assertEqual(b'{"data":[]}', result.read()) + self.assertEqual('Bearer private-upstream-test-key', upstream.call_args.args[0].get_header('Authorization')) + connection.close() + + def test_authorized_post_forwards_json_unchanged(self): + relay.KEY = 'private-upstream-test-key' + body = b'{"model":"test-model","messages":[],"stream":true}' + response = unittest.mock.Mock(status=200, headers={'Content-Type': 'text/event-stream'}) + response.read1.side_effect = [b'data: [DONE]\n', b''] + connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) + with patch.object(relay.UPSTREAM, 'open', return_value=response) as upstream, patch.object(relay, 'record'): + connection.request('POST', '/v1/chat/completions', body=body, + headers={'Authorization': 'Bearer ' + relay.TOKEN}) + result = connection.getresponse() + self.assertEqual(200, result.status) + self.assertEqual(b'data: [DONE]\n', result.read()) + self.assertEqual(body, upstream.call_args.args[0].data) + self.assertEqual('POST', upstream.call_args.args[0].get_method()) + connection.close() + + +class RedirectTest(unittest.TestCase): + def test_redirects_never_reach_destination(self): + class RedirectServer(http.server.BaseHTTPRequestHandler): + paths = [] + + def log_message(self, *_): + pass + + def do_GET(self): + self.paths.append(self.path) + self.send_response(int(self.path[1:])) + self.send_header('Location', '/destination') + self.send_header('Content-Length', '0') + self.end_headers() + + server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), RedirectServer) + worker = threading.Thread(target=server.serve_forever) + worker.start() + try: + for status in [301, 302, 303, 307, 308]: + with self.subTest(status=status): + request = relay.urllib.request.Request( + 'http://127.0.0.1:' + str(server.server_port) + '/' + str(status), + headers={'Authorization': 'Bearer test-only-key'}) + with self.assertRaises(urllib.error.HTTPError) as caught: + relay.UPSTREAM.open(request, timeout=5) + self.assertEqual(status, caught.exception.code) + caught.exception.close() + self.assertEqual(['/301', '/302', '/303', '/307', '/308'], RedirectServer.paths) + finally: + server.shutdown() + server.server_close() + worker.join() + + +if __name__ == '__main__': + unittest.main() diff --git a/jetbrains_plugin/scripts/ui-harness/re-delegate/test_streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/re-delegate/test_streaming_fixture.py new file mode 100644 index 0000000..72f5a85 --- /dev/null +++ b/jetbrains_plugin/scripts/ui-harness/re-delegate/test_streaming_fixture.py @@ -0,0 +1,188 @@ +import http.client +import json +import tempfile +import threading +import unittest +from unittest.mock import patch +from pathlib import Path + +from streaming_fixture import history_role, make_server + + +class StreamingFixtureTest(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.log = Path(self.directory.name) / 'provider.jsonl' + self.server = make_server(0, self.log, chunks=3, interval=0.03) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + self.connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port, timeout=5) + + def tearDown(self): + self.connection.close() + self.server.shutdown() + self.server.server_close() + self.thread.join() + self.directory.cleanup() + + def post(self, body, token='focus-fixture-local-only'): + self.connection.request('POST', '/v1/chat/completions', json.dumps(body), + {'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json'}) + return self.connection.getresponse() + + def test_two_incremental_streams_finish_with_native_completion(self): + request = {'stream': True, 'messages': [], 'tools': [ + {'type': 'function', 'function': {'name': 'attempt_completion'}}]} + for turn in (1, 2): + paused = threading.Event() + release = threading.Event() + + def pause_stream(_interval): + paused.set() + if not release.wait(5): + raise TimeoutError('Test did not release the fixture stream') + + # Hold the server after its first content chunk. Receiving both the + # role and content before release proves incremental delivery without + # depending on sleep durations or the host's clock resolution. + with patch('streaming_fixture.time.sleep', side_effect=pause_stream): + try: + response = self.post(request) + self.assertEqual(200, response.status) + self.assertEqual('text/event-stream', response.getheader('Content-Type')) + first = json.loads(response.readline().decode()[6:]) + self.assertEqual('assistant', first['choices'][0]['delta']['role']) + self.assertEqual(b'\n', response.readline()) + update = json.loads(response.readline().decode()[6:]) + self.assertEqual(f'History update 1 for turn {turn}. ', + update['choices'][0]['delta']['content']) + self.assertTrue(paused.wait(5), 'Server did not pause during the stream') + finally: + # Also unblock the request handler if any assertion fails. + release.set() + events = [update] + [json.loads(line[6:]) for line in response.read().decode().splitlines() + if line.startswith('data: {')] + updates = [e for e in events if e['choices'][0]['delta'].get('content')] + self.assertEqual(3, len(updates)) + tool = events[-2]['choices'][0]['delta']['tool_calls'][0] + self.assertEqual('attempt_completion', tool['function']['name']) + self.assertIn(str(turn), json.loads(tool['function']['arguments'])['result']) + self.assertEqual('tool_calls', events[-1]['choices'][0]['finish_reason']) + records = [json.loads(line) for line in self.log.read_text().splitlines()] + self.assertEqual(2, len([e for e in records if e['event'] == 'completed'])) + + def test_rejects_non_streaming_and_missing_completion_tool(self): + for request in ({'stream': False}, {'stream': True, 'tools': []}): + response = self.post(request) + self.assertEqual(400, response.status) + response.read() + self.assertEqual(0, self.server.requests) + + def test_rejects_invalid_local_token(self): + response = self.post({'stream': True}, token='wrong') + self.assertEqual(401, response.status) + response.read() + self.assertEqual(0, self.server.requests) + + def test_health_and_model_discovery(self): + self.connection.request('GET', '/health') + response = self.connection.getresponse() + self.assertEqual(b'ready', response.read()) + self.connection.request('GET', '/v1/models') + response = self.connection.getresponse() + self.assertEqual('focus-fixture', json.loads(response.read())['data'][0]['id']) + + +class HistoryRaceScenarioTest(unittest.TestCase): + def user(self, text): + return {'role': 'user', 'content': [{'type': 'text', 'text': text}]} + + def test_roles_follow_the_first_user_message(self): + self.assertEqual(('delegate', 'A'), history_role([self.user('HISTORY_RACE_PARENT A go')])) + self.assertEqual(('child', 'B'), history_role([self.user('HISTORY_RACE_CHILD B go')])) + self.assertEqual(('plain', ''), history_role([self.user('hello')])) + + def test_parent_resumes_after_new_task_but_ignores_the_system_prompt(self): + system = {'role': 'system', 'content': 'tools: new_task, attempt_completion'} + first = self.user('HISTORY_RACE_PARENT A go') + self.assertEqual(('delegate', 'A'), history_role([system, first])) + called = {'role': 'assistant', 'tool_calls': [{'function': {'name': 'new_task'}}]} + self.assertEqual(('resume', 'A'), history_role([system, first, called])) + + def test_two_parents_delegate_together_and_children_complete(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + log = Path(directory.name) / 'provider.jsonl' + server = make_server(0, log, chunks=2, interval=0.01, scenario='history-race', barrier=2) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + tools = [{'type': 'function', 'function': {'name': 'attempt_completion'}}] + results = {} + + def call(tag, text): + connection = http.client.HTTPConnection('127.0.0.1', server.server_port, timeout=10) + body = {'stream': True, 'tools': tools, 'messages': [self.user(text)]} + connection.request('POST', '/v1/chat/completions', json.dumps(body), + {'Authorization': 'Bearer focus-fixture-local-only'}) + data = connection.getresponse().read().decode() + results[tag] = [json.loads(line[6:]) for line in data.splitlines() if line.startswith('data: {')] + connection.close() + + def events(): + return [json.loads(line)['event'] for line in log.read_text().splitlines()] if log.exists() else [] + + parents = {tag: threading.Thread(target=call, args=(tag, f'HISTORY_RACE_PARENT {tag}')) for tag in 'AB'} + # Parent A alone must stay blocked: no new_task and no barrier-released. + parents['A'].start() + parents['A'].join(0.5) + self.assertTrue(parents['A'].is_alive()) + self.assertNotIn('A', results) + self.assertNotIn('barrier-released', events()) + parents['B'].start() + for parent in parents.values(): + parent.join(15) + self.assertFalse(parent.is_alive()) + for tag in 'AB': + call_delta = results[tag][-2]['choices'][0]['delta']['tool_calls'][0]['function'] + self.assertEqual('new_task', call_delta['name']) + arguments = json.loads(call_delta['arguments']) + self.assertEqual(f'HISTORY_RACE_CHILD {tag} reply with attempt_completion.', arguments['message']) + self.assertIsNone(arguments['todos']) + call('A', 'HISTORY_RACE_CHILD A go') + done = results['A'][-2]['choices'][0]['delta']['tool_calls'][0]['function']['name'] + self.assertEqual('attempt_completion', done) + events = [json.loads(line)['event'] for line in log.read_text().splitlines()] + self.assertEqual(2, events.count('barrier-released')) + self.assertNotIn('barrier-timeout', events) + counts = [json.loads(line)['chunks'] for line in log.read_text().splitlines() + if json.loads(line)['event'] == 'completed'] + self.assertEqual([2, 3, 3], sorted(counts)) + + def test_lone_parent_times_out_instead_of_passing(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + log = Path(directory.name) / 'provider.jsonl' + server = make_server(0, log, chunks=1, interval=0.01, scenario='history-race', + barrier=2, barrier_timeout=0.3) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + connection = http.client.HTTPConnection('127.0.0.1', server.server_port, timeout=10) + body = {'stream': True, 'tools': [{'type': 'function', 'function': {'name': 'attempt_completion'}}], + 'messages': [self.user('HISTORY_RACE_PARENT A go')]} + connection.request('POST', '/v1/chat/completions', json.dumps(body), + {'Authorization': 'Bearer focus-fixture-local-only'}) + connection.getresponse().read() + connection.close() + events = [json.loads(line)['event'] for line in log.read_text().splitlines()] + self.assertIn('barrier-timeout', events) + self.assertNotIn('barrier-released', events) + + +if __name__ == '__main__': + unittest.main() diff --git a/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 b/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 index 890247e..b64b81d 100644 --- a/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 +++ b/jetbrains_plugin/scripts/ui-harness/run-windows.ps1 @@ -17,6 +17,10 @@ $ErrorActionPreference = 'Stop' $IdeDir = (Resolve-Path $IdeDir).Path $PluginZip = (Resolve-Path $PluginZip).Path $EvidenceDir = [IO.Path]::GetFullPath($EvidenceDir) +if ($HistoryRace -and $env:GITHUB_ACTIONS -ne 'true') { + # This mode shares the real user profile store. Copies of its tasks go to the evidence. + throw 'The history race test uses the real user profile. Run it only on a disposable CI runner.' +} New-Item -ItemType Directory -Force $EvidenceDir | Out-Null if (Test-Path (Join-Path $EvidenceDir 'status.txt')) { throw 'Use a fresh evidence directory for each run.' } $runDir = Join-Path ([IO.Path]::GetTempPath()) ('zoo-focus-' + [guid]::NewGuid().ToString('N')) @@ -25,10 +29,6 @@ $ideProcess = $null $instances = @{} $fixtureProcess = $null $live = $Provider -eq 'solheim' -if ($HistoryRace -and $env:GITHUB_ACTIONS -ne 'true') { - # This mode shares the real user profile store. Copies of its tasks go to the evidence. - throw 'The history race test uses the real user profile. Run it only on a disposable CI runner.' -} if ($HistoryRace -and $live) { throw 'The history race test uses only the local fixture provider.' } $originalRelayToken = $env:SOLHEIM_RELAY_TOKEN $originalKeyFile = $env:SOLHEIM_KEY_FILE diff --git a/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py b/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py index ff2a459..b095026 100644 --- a/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py +++ b/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py @@ -214,6 +214,9 @@ def provider_failures(events): provider_failures(['barrier-released'] + ['completed'] * 6)) self.assertIn('5 completed streams, expected 6', provider_failures(['barrier-released'] * 2 + ['completed'] * 5)) + self.assertIn('7 completed streams, expected 6', provider_failures(good + ['completed'])) + self.assertIn('barrier released 3 times, expected 2', provider_failures(good + ['barrier-released'])) + self.assertEqual(['provider logged barrier-timeout'], provider_failures(good + ['barrier-timeout'])) provider.write_text('{"event": "completed"}\n{broken') self.assertTrue(any('not valid JSON' in f for f in check_provider(provider, 2))) provider.write_text('[1]\n' + '\n'.join(json.dumps({'event': e}) for e in good)) @@ -223,6 +226,9 @@ def provider_failures(events): log = Path(directory) / 'idea.log' log.write_text('ok\nInvalid status transition interrupted -> delegated\n') self.assertEqual(1, len(check_logs([log]))) + for phrase in ('invalid status transition', 'DeltaRejectedError', 'failed to persist', 'Cannot re-delegate'): + log.write_text(f'ok\n{phrase}\n') + self.assertEqual([f'idea.log:2: {phrase}'], check_logs([log]), phrase) if __name__ == '__main__': diff --git a/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py index 0a825e6..23428d4 100644 --- a/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py +++ b/jetbrains_plugin/scripts/ui-harness/test_streaming_fixture.py @@ -122,9 +122,10 @@ def test_two_parents_delegate_together_and_children_complete(self): tools = [{'type': 'function', 'function': {'name': 'attempt_completion'}}] results = {} - def call(tag, text): + def call(tag, text, extra=()): + extra = list(extra) connection = http.client.HTTPConnection('127.0.0.1', server.server_port, timeout=10) - body = {'stream': True, 'tools': tools, 'messages': [self.user(text)]} + body = {'stream': True, 'tools': tools, 'messages': [self.user(text)] + extra} connection.request('POST', '/v1/chat/completions', json.dumps(body), {'Authorization': 'Bearer focus-fixture-local-only'}) data = connection.getresponse().read().decode() @@ -154,9 +155,14 @@ def events(): call('A', 'HISTORY_RACE_CHILD A go') done = results['A'][-2]['choices'][0]['delta']['tool_calls'][0]['function']['name'] self.assertEqual('attempt_completion', done) + called = {'role': 'assistant', 'tool_calls': [{'function': {'name': 'new_task'}}]} + call('A', 'HISTORY_RACE_PARENT A go', [called]) events = [json.loads(line)['event'] for line in log.read_text().splitlines()] self.assertEqual(2, events.count('barrier-released')) self.assertNotIn('barrier-timeout', events) + counts = [json.loads(line)['chunks'] for line in log.read_text().splitlines() + if json.loads(line)['event'] == 'completed'] + self.assertEqual([2, 3, 3, 3], sorted(counts)) def test_lone_parent_times_out_instead_of_passing(self): directory = tempfile.TemporaryDirectory() From 64be7d5e2504acbe1e6e36da5c57016759cf912e Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 16 Nov 2026 01:14:46 +0000 Subject: [PATCH 5/5] test: remove stray directories and close log and childIds test gaps --- jetbrains_plugin/scripts/ui-harness/README.md | 2 +- .../ui-harness/persist/check_history_store.py | 143 ----------- .../ui-harness/persist/solheim_proxy.py | 145 ----------- .../ui-harness/persist/streaming_fixture.py | 157 ------------ .../persist/test_check_history_store.py | 235 ------------------ .../ui-harness/persist/test_solheim_proxy.py | 156 ------------ .../persist/test_streaming_fixture.py | 188 -------------- .../re-delegate/check_history_store.py | 143 ----------- .../ui-harness/re-delegate/solheim_proxy.py | 145 ----------- .../re-delegate/streaming_fixture.py | 157 ------------ .../re-delegate/test_check_history_store.py | 235 ------------------ .../re-delegate/test_solheim_proxy.py | 156 ------------ .../re-delegate/test_streaming_fixture.py | 188 -------------- .../ui-harness/test_check_history_store.py | 9 + 14 files changed, 10 insertions(+), 2049 deletions(-) delete mode 100644 jetbrains_plugin/scripts/ui-harness/persist/check_history_store.py delete mode 100644 jetbrains_plugin/scripts/ui-harness/persist/solheim_proxy.py delete mode 100644 jetbrains_plugin/scripts/ui-harness/persist/streaming_fixture.py delete mode 100644 jetbrains_plugin/scripts/ui-harness/persist/test_check_history_store.py delete mode 100644 jetbrains_plugin/scripts/ui-harness/persist/test_solheim_proxy.py delete mode 100644 jetbrains_plugin/scripts/ui-harness/persist/test_streaming_fixture.py delete mode 100644 jetbrains_plugin/scripts/ui-harness/re-delegate/check_history_store.py delete mode 100644 jetbrains_plugin/scripts/ui-harness/re-delegate/solheim_proxy.py delete mode 100644 jetbrains_plugin/scripts/ui-harness/re-delegate/streaming_fixture.py delete mode 100644 jetbrains_plugin/scripts/ui-harness/re-delegate/test_check_history_store.py delete mode 100644 jetbrains_plugin/scripts/ui-harness/re-delegate/test_solheim_proxy.py delete mode 100644 jetbrains_plugin/scripts/ui-harness/re-delegate/test_streaming_fixture.py diff --git a/jetbrains_plugin/scripts/ui-harness/README.md b/jetbrains_plugin/scripts/ui-harness/README.md index 0f319de..5eeb3b0 100644 --- a/jetbrains_plugin/scripts/ui-harness/README.md +++ b/jetbrains_plugin/scripts/ui-harness/README.md @@ -56,7 +56,7 @@ It then streams both child tasks and lets each parent resume and finish. `check_ 3. Each parent lists only its own child, and each child names only its own parent. 4. Each child is `completed` and no parent still waits on a child. 5. The provider log shows both barriers released, no disconnect, and six completed streams. -6. The IDE logs hold no invalid status transition or failed persist message. +6. The IDE logs hold no invalid status transition, rejected delta, failed persist, or re-delegate error message. The driver also fails if the webview changes task more than three times. The expected order is parent, child, parent. The test counts task switches only. It does not detect a diff --git a/jetbrains_plugin/scripts/ui-harness/persist/check_history_store.py b/jetbrains_plugin/scripts/ui-harness/persist/check_history_store.py deleted file mode 100644 index b76de99..0000000 --- a/jetbrains_plugin/scripts/ui-harness/persist/check_history_store.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python3 -"""Check the shared Zoo Code task history after a concurrent two-IDE run. - -Issue #6: several IDE processes write one global history store. A safe store -keeps one valid history_item.json per task, with no shared index file and with -parent and child links that stay inside one IDE's task tree. -""" -import argparse -import json -import re -import sys -from pathlib import Path - -PLACEHOLDER = re.compile(r'^\s*(Task|Work) #\d+') -BAD_LOG = re.compile(r'invalid status transition|DeltaRejectedError|failed to persist|Cannot re-delegate', re.I) -MARKER = re.compile(r'HISTORY_RACE_(PARENT|CHILD) (\w+)') - - -def load_items(storage, since): - """Return ({task id: item}, failures) for tasks written at or after `since`.""" - items, failures = {}, [] - for index in storage.glob('**/tasks/_index.json'): - if index.stat().st_mtime >= since: - failures.append(f'shared index file was written: {index}') - for path in sorted(storage.glob('**/tasks/*/history_item.json')): - if path.stat().st_mtime < since: - continue - try: - item = json.loads(path.read_text(encoding='utf-8')) - except (OSError, ValueError) as error: - failures.append(f'{path.parent.name}: history_item.json is not valid JSON ({error})') - continue - if not isinstance(item, dict): - failures.append(f'{path.parent.name}: history_item.json is not an object') - continue - if item.get('id') != path.parent.name: - failures.append(f'{path.parent.name}: id field is {item.get("id")!r}') - continue - items[item['id']] = item - return items, failures - - -def check_items(items, instances): - failures = [] - tasks = {} - for task_id, item in items.items(): - text = item.get('task') or '' - if PLACEHOLDER.match(text) or not text.strip(): - failures.append(f'{task_id}: placeholder or empty title {text!r}') - continue - match = MARKER.search(text) - if not match: - failures.append(f'{task_id}: title does not carry a run marker: {text!r}') - continue - if not item.get('size'): - failures.append(f'{task_id}: size is {item.get("size")!r} after real work') - tasks[task_id] = (match.group(1), match.group(2), item) - parents = {i: t for i, t in tasks.items() if t[0] == 'PARENT'} - children = {i: t for i, t in tasks.items() if t[0] == 'CHILD'} - if len(parents) != instances or len(children) != instances: - failures.append(f'expected {instances} parents and {instances} children, ' - f'found {len(parents)} and {len(children)}') - for parent_id, (_, tag, parent) in parents.items(): - own = [c for c, t in children.items() if t[1] == tag] - if len(own) != 1: - failures.append(f'{parent_id}: expected one child for tag {tag}, found {len(own)}') - continue - child_id = own[0] - child = children[child_id][2] - if parent.get('childIds') != [child_id]: - failures.append(f'{parent_id}: childIds {parent.get("childIds")!r}, expected {[child_id]!r}') - if child.get('parentTaskId') != parent_id: - failures.append(f'{child_id}: parentTaskId {child.get("parentTaskId")!r}, expected {parent_id!r}') - if child.get('status') != 'completed': - failures.append(f'{child_id}: child status {child.get("status")!r}, expected completed') - if parent.get('status') == 'delegated' or parent.get('awaitingChildId') or parent.get('delegatedToId'): - failures.append(f'{parent_id}: parent still waits on a child (status {parent.get("status")!r})') - if parent.get('workspace') != child.get('workspace'): - failures.append(f'{parent_id}: parent and child workspaces differ') - workspaces = {t[2].get('workspace') for t in parents.values()} - if instances > 1 and len(workspaces) != instances: - failures.append(f'expected {instances} distinct workspaces, found {sorted(map(str, workspaces))}') - return failures - - -def check_provider(path, instances): - events, failures = [], [] - for number, line in enumerate(path.read_text(encoding='utf-8').splitlines(), 1): - if not line.strip(): - continue - try: - events.append(json.loads(line)) - except ValueError as error: - failures.append(f'{path.name}:{number}: provider log line is not valid JSON ({error})') - for event in events: - if not isinstance(event, dict): - failures.append(f'{path.name}: provider log line is not an object: {event!r}') - names = [e.get('event') for e in events if isinstance(e, dict)] - if names.count('barrier-released') != instances: - failures.append(f'barrier released {names.count("barrier-released")} times, expected {instances}') - for bad in ('barrier-timeout', 'disconnected'): - if bad in names: - failures.append(f'provider logged {bad}') - if names.count('completed') != 3 * instances: - failures.append(f'{names.count("completed")} completed streams, expected {3 * instances}') - return failures - - -def check_logs(paths): - failures = [] - for path in paths: - if not path.exists(): - failures.append(f'IDE log does not exist: {path}') - continue - for number, line in enumerate(path.read_text(encoding='utf-8', errors='replace').splitlines(), 1): - if BAD_LOG.search(line): - failures.append(f'{path.name}:{number}: {line.strip()[:200]}') - return failures - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--storage', type=Path, required=True) - parser.add_argument('--since', type=float, required=True, help='epoch seconds when the run started') - parser.add_argument('--provider-log', type=Path, required=True) - parser.add_argument('--ide-log', type=Path, action='append', required=True) - parser.add_argument('--instances', type=int, default=2) - parser.add_argument('--report', type=Path) - args = parser.parse_args() - items, failures = load_items(args.storage, args.since) - failures += check_items(items, args.instances) - failures += check_provider(args.provider_log, args.instances) - failures += check_logs(args.ide_log) - report = {'tasks': len(items), 'failures': failures} - text = json.dumps(report, indent=2) - if args.report: - args.report.write_text(text, encoding='utf-8') - print(text) - return 1 if failures else 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/jetbrains_plugin/scripts/ui-harness/persist/solheim_proxy.py b/jetbrains_plugin/scripts/ui-harness/persist/solheim_proxy.py deleted file mode 100644 index 669102d..0000000 --- a/jetbrains_plugin/scripts/ui-harness/persist/solheim_proxy.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python3 -"""Loopback authentication relay for manual Solheim extension focus smoke tests. - -Real credentials stay in SOLHEIM_KEY_FILE (mode 0600), outside IDE settings/logs. -The extension uses a per-run relay token and http://127.0.0.1:19487/v1. Logs contain only -request metadata, upstream status, stream byte counts, and tool names. -""" -import http.server -import hmac -import json -import os -import time -import urllib.error -import urllib.request -from pathlib import Path - -KEY = None -TOKEN = None -BASE = 'https://api.solheim.ai/v1' -MAX_BODY_BYTES = 8 * 1024 * 1024 -LOG = Path(os.environ.get('SOLHEIM_PROXY_LOG', '/tmp/zoo-solheim-proxy.jsonl')) - - -def record(**fields): - with LOG.open('a') as output: - output.write(json.dumps({'time': time.time(), **fields}) + '\n') - - -class RefuseRedirects(urllib.request.HTTPRedirectHandler): - def redirect_request(self, request, response, code, message, headers, new_url): - # Never send provider credentials to a redirected URL. - return None - - -UPSTREAM = urllib.request.build_opener(RefuseRedirects()) - - -class Relay(http.server.BaseHTTPRequestHandler): - protocol_version = 'HTTP/1.1' - - def log_message(self, *_): - pass - - def do_GET(self): - self.relay() - - def do_POST(self): - self.relay() - - def relay(self): - expected_host = '127.0.0.1:' + str(self.server.server_port) - if self.headers.get('Host') != expected_host or self.headers.get('Origin') is not None: - self.send_error(403) - return - supplied = self.headers.get('Authorization', '') - if not TOKEN or not hmac.compare_digest(supplied, 'Bearer ' + TOKEN): - self.send_error(401) - return - if not self.path.startswith('/v1/'): - self.send_error(404) - return - if self.command == 'GET' and self.path == '/v1/health': - # Local readiness only; never probe the paid upstream in a startup loop. - self.send_response(200) - self.send_header('Content-Length', '5') - self.end_headers() - self.wfile.write(b'ready') - return - lengths = self.headers.get_all('Content-Length', []) - length_text = lengths[0] if lengths else '0' - if (len(lengths) > 1 or self.headers.get('Transfer-Encoding') is not None - or not length_text.isascii() or not length_text.isdecimal()): - self.send_error(400, 'Invalid request framing') - return - # Check before int conversion, including absurdly long decimal strings. - if len(length_text.lstrip('0')) > 8 or int(length_text.lstrip('0') or '0') > MAX_BODY_BYTES: - self.send_error(413, 'Request body too large') - return - length = int(length_text.lstrip('0') or '0') - body = self.rfile.read(length) - try: - parsed = json.loads(body) if body else {} - if len(body) != length or not isinstance(parsed, dict): - raise ValueError('Expected a JSON object') - if not isinstance(parsed.get('messages', []), list): - raise ValueError('Expected a message list') - except (ValueError, UnicodeError): - self.send_error(400, 'Invalid JSON request body') - return - record(event='request', path=self.path, model=parsed.get('model'), - messages=len(parsed.get('messages', [])), stream=parsed.get('stream', False)) - request = urllib.request.Request(BASE + self.path[3:], data=body or None, - method=self.command, - headers={'Authorization': 'Bearer ' + KEY, - 'Content-Type': 'application/json'}) - try: - response = UPSTREAM.open(request, timeout=180) - except urllib.error.HTTPError as error: - response = error - except Exception as error: - record(event='transport-error', error=type(error).__name__) - self.send_error(502) - return - self.send_response(response.status) - self.send_header('Content-Type', response.headers.get('Content-Type', 'application/json')) - self.send_header('Connection', 'close') - self.end_headers() - count = 0 - buffer = b'' - tools = set() - try: - while True: - chunk = response.read1(4096) - if not chunk: - break - count += len(chunk) - self.wfile.write(chunk) - self.wfile.flush() - buffer += chunk - while b'\n' in buffer: - line, buffer = buffer.split(b'\n', 1) - if line.startswith(b'data: ') and line != b'data: [DONE]': - try: - event = json.loads(line[6:]) - for choice in event.get('choices', []): - for tool in choice.get('delta', {}).get('tool_calls', []): - name = tool.get('function', {}).get('name') - if name: - tools.add(name) - except (ValueError, TypeError): - pass - except (BrokenPipeError, ConnectionResetError): - record(event='client-disconnected') - finally: - response.close() - self.close_connection = True - record(event='response', status=response.status, bytes=count, tools=sorted(tools)) - - -if __name__ == '__main__': - KEY = Path(os.environ['SOLHEIM_KEY_FILE']).read_text().strip() - TOKEN = os.environ['SOLHEIM_RELAY_TOKEN'] - if len(TOKEN) < 32: - raise SystemExit('Use a random relay token of at least 32 characters') - http.server.ThreadingHTTPServer(('127.0.0.1', 19487), Relay).serve_forever() diff --git a/jetbrains_plugin/scripts/ui-harness/persist/streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/persist/streaming_fixture.py deleted file mode 100644 index 07d7af8..0000000 --- a/jetbrains_plugin/scripts/ui-harness/persist/streaming_fixture.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic loopback OpenAI fixture for headful chat-focus CI tests. - -Only the provider is simulated. The extension, React history, JCEF, and native -keyboard delivery remain real. No commands or file edits are requested. -""" -import argparse -import http.server -import json -import re -import threading -import time -from pathlib import Path - - -def _text(content): - if isinstance(content, list): - return ''.join(part.get('text', '') for part in content if isinstance(part, dict)) - return content if isinstance(content, str) else '' - - -def history_role(messages): - """Classify a request in the history-race scenario. - - The first user message names the task: a parent carries HISTORY_RACE_PARENT - and a child carries HISTORY_RACE_CHILD . A parent that already holds a - new_task call in a non-system message has delegated, so it resumes. - """ - users = [m for m in messages if m.get('role') == 'user'] - first = _text(users[0].get('content')) if users else '' - child = re.search(r'HISTORY_RACE_CHILD (\w+)', first) - if child: - return 'child', child.group(1) - parent = re.search(r'HISTORY_RACE_PARENT (\w+)', first) - if not parent: - return 'plain', '' - body = json.dumps([m for m in messages if m.get('role') != 'system']) - return ('resume' if 'new_task' in body else 'delegate'), parent.group(1) - - -class Fixture(http.server.BaseHTTPRequestHandler): - def log_message(self, *_): - pass - - def do_GET(self): - if self.path == '/health': - self.send_response(200) - self.end_headers() - self.wfile.write(b'ready') - elif self.path == '/v1/models': - self.send_response(200) - self.send_header('Content-Type', 'application/json') - self.end_headers() - self.wfile.write(json.dumps({'object': 'list', 'data': [ - {'id': 'focus-fixture', 'object': 'model', 'owned_by': 'fixture'} - ]}).encode()) - else: - self.send_error(404) - - def do_POST(self): - if self.path != '/v1/chat/completions': - self.send_error(404) - return - if self.headers.get('Authorization') != 'Bearer focus-fixture-local-only': - self.send_error(401) - return - try: - length = int(self.headers.get('Content-Length', '0')) - if not 0 < length <= 8 * 1024 * 1024: - raise ValueError('body size') - request = json.loads(self.rfile.read(length)) - if request.get('stream') is not True: - raise ValueError('streaming required') - tools = [t.get('function', {}).get('name') for t in request.get('tools', [])] - if 'attempt_completion' not in tools: - raise ValueError('native attempt_completion tool required') - except (ValueError, TypeError, AttributeError): - self.send_error(400, 'Expected streaming request with native completion tool') - return - with self.server.counter_lock: - self.server.requests += 1 - turn = self.server.requests - role, tag = 'plain', '' - if self.server.scenario == 'history-race': - role, tag = history_role(request.get('messages', [])) - self.server.record(event='request', turn=turn, tools=tools, role=role, tag=tag) - self.send_response(200) - self.send_header('Content-Type', 'text/event-stream') - self.send_header('Cache-Control', 'no-cache') - self.end_headers() - - def emit(delta, finish=None): - chunk = {'id': f'focus-{turn}', 'object': 'chat.completion.chunk', - 'created': int(time.time()), 'model': 'focus-fixture', - 'choices': [{'index': 0, 'delta': delta, 'finish_reason': finish}]} - self.wfile.write(('data: ' + json.dumps(chunk) + '\n\n').encode()) - self.wfile.flush() - - try: - emit({'role': 'assistant', 'content': ''}) - chunks = self.server.chunks - if role == 'delegate': - # Hold every parent until all IDEs have submitted, so the delegations - # and the child streams that follow overlap in time. - try: - self.server.barrier.wait() - except threading.BrokenBarrierError: - self.server.record(event='barrier-timeout', turn=turn, tag=tag) - return - self.server.record(event='barrier-released', turn=turn, tag=tag) - if role in ('delegate', 'resume'): - chunks = 3 - for index in range(chunks): - emit({'content': f'History update {index + 1} for turn {turn}. '}) - time.sleep(self.server.interval) - if role == 'delegate': - call = ('new_task', {'mode': 'code', 'todos': None, - 'message': f'HISTORY_RACE_CHILD {tag} reply with attempt_completion.'}) - else: - call = ('attempt_completion', {'result': f'Fixture turn {turn} complete.'}) - emit({'tool_calls': [{'index': 0, 'id': f'call-{turn}', - 'type': 'function', 'function': { - 'name': call[0], 'arguments': json.dumps(call[1])}}]}) - emit({}, 'tool_calls') - self.wfile.write(b'data: [DONE]\n\n') - self.wfile.flush() - self.server.record(event='completed', turn=turn, chunks=chunks) - except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError): - self.server.record(event='disconnected', turn=turn) - - -def make_server(port=19487, log=None, chunks=100, interval=0.25, scenario=None, barrier=2, barrier_timeout=300): - server = http.server.ThreadingHTTPServer(('127.0.0.1', port), Fixture) - server.requests = 0 - server.counter_lock = threading.Lock() - server.chunks, server.interval = chunks, interval - server.scenario = scenario - server.barrier = threading.Barrier(barrier, timeout=barrier_timeout) - - def record(**fields): - if log: - with server.counter_lock, Path(log).open('a', encoding='utf-8') as output: - output.write(json.dumps({'time': time.time(), **fields}) + '\n') - server.record = record - return server - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--port', type=int, default=19487) - parser.add_argument('--log', type=Path, required=True) - parser.add_argument('--scenario', choices=['history-race']) - parser.add_argument('--chunks', type=int, default=100) - parser.add_argument('--interval', type=float, default=0.25) - parser.add_argument('--barrier', type=int, default=2) - args = parser.parse_args() - make_server(args.port, args.log, args.chunks, args.interval, args.scenario, args.barrier).serve_forever() diff --git a/jetbrains_plugin/scripts/ui-harness/persist/test_check_history_store.py b/jetbrains_plugin/scripts/ui-harness/persist/test_check_history_store.py deleted file mode 100644 index b095026..0000000 --- a/jetbrains_plugin/scripts/ui-harness/persist/test_check_history_store.py +++ /dev/null @@ -1,235 +0,0 @@ -import json -import tempfile -import unittest -from pathlib import Path - -from check_history_store import check_items, check_logs, check_provider, load_items - - -def item(task_id, text, **extra): - return {'id': task_id, 'task': text, 'size': 10, 'workspace': 'w', **extra} - - -def good_pair(tag, workspace): - return { - 'p' + tag: item('p' + tag, f'HISTORY_RACE_PARENT {tag} go', childIds=['c' + tag], - status='active', workspace=workspace), - 'c' + tag: item('c' + tag, f'HISTORY_RACE_CHILD {tag} go', parentTaskId='p' + tag, - status='completed', workspace=workspace), - } - - -# Real items from Windows CI run 43562237505 (commit 5a494cf, IDEA 2026.2.1, extension 3.96.0). -# Only the temporary run folder name in `workspace` was shortened. -REAL_ITEMS = json.loads(r'''[ - { - "id": "01a1d898-c7ab-70da-936e-6467341e0125", - "parentTaskId": "01a1d898-bf28-71a1-92d9-df474bf72871", - "number": 1, - "ts": 1794635304779, - "task": "HISTORY_RACE_CHILD A reply with attempt_completion.", - "tokensIn": 0, - "tokensOut": 0, - "totalCost": 0, - "size": 10621, - "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-A", - "mode": "code", - "apiConfigName": "focus-fixture", - "status": "completed", - "completionResultSummary": "Fixture turn 4 complete." - }, - { - "id": "01a1d898-c663-72e9-b6dd-1caba4bda93a", - "parentTaskId": "01a1d898-4435-75fd-afc2-6ca3aa4c89a4", - "number": 1, - "ts": 1794635304331, - "task": "HISTORY_RACE_CHILD B reply with attempt_completion.", - "tokensIn": 0, - "tokensOut": 0, - "totalCost": 0, - "size": 10623, - "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-B", - "mode": "code", - "apiConfigName": "focus-fixture", - "status": "completed", - "completionResultSummary": "Fixture turn 3 complete." - }, - { - "id": "01a1d898-bf28-71a1-92d9-df474bf72871", - "number": 1, - "ts": 1794635306508, - "task": "HISTORY_RACE_PARENT A delegate one subtask, then finish.", - "tokensIn": 0, - "tokensOut": 0, - "totalCost": 0, - "size": 7694, - "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-A", - "mode": "code", - "apiConfigName": "focus-fixture", - "status": "active", - "childIds": [ - "01a1d898-c7ab-70da-936e-6467341e0125" - ], - "completedByChildId": "01a1d898-c7ab-70da-936e-6467341e0125", - "completionResultSummary": "Fixture turn 4 complete." - }, - { - "id": "01a1d898-4435-75fd-afc2-6ca3aa4c89a4", - "number": 1, - "ts": 1794635306029, - "task": "HISTORY_RACE_PARENT B delegate one subtask, then finish.", - "tokensIn": 0, - "tokensOut": 0, - "totalCost": 0, - "size": 7698, - "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-B", - "mode": "code", - "apiConfigName": "focus-fixture", - "status": "active", - "childIds": [ - "01a1d898-c663-72e9-b6dd-1caba4bda93a" - ], - "completedByChildId": "01a1d898-c663-72e9-b6dd-1caba4bda93a", - "completionResultSummary": "Fixture turn 3 complete." - } -]''') - - -class CheckItemsTest(unittest.TestCase): - def test_real_extension_items_pass(self): - items = {i['id']: i for i in REAL_ITEMS} - self.assertEqual(4, len(items)) - self.assertEqual([], check_items(items, 2)) - - - def two(self): - return {**good_pair('A', 'wa'), **good_pair('B', 'wb')} - - def test_clean_store_passes(self): - self.assertEqual([], check_items(self.two(), 2)) - - def test_placeholder_title_and_zero_size_fail(self): - items = self.two() - items['pA']['task'] = 'Task #1' - items['cB']['size'] = 0 - failures = check_items(items, 2) - self.assertTrue(any('placeholder' in f for f in failures)) - self.assertTrue(any('size is 0' in f for f in failures)) - - def test_cross_linked_children_fail(self): - items = self.two() - items['pA']['childIds'] = ['cB'] - items['cA']['parentTaskId'] = 'pB' - failures = check_items(items, 2) - self.assertTrue(any('childIds' in f for f in failures)) - self.assertTrue(any('parentTaskId' in f for f in failures)) - - def test_stuck_delegation_and_open_child_fail(self): - items = self.two() - items['pA'].update(status='delegated', awaitingChildId='cA') - items['cA']['status'] = 'interrupted' - failures = check_items(items, 2) - self.assertTrue(any('still waits' in f for f in failures)) - self.assertTrue(any('child status' in f for f in failures)) - - def fails(self, items, message): - self.assertIn(message, check_items(items, 2)) - - def test_each_bad_item_branch_reports_an_exact_message(self): - items = self.two() - items['pA']['task'] = 'Work #2' - self.fails(items, "pA: placeholder or empty title 'Work #2'") - items = self.two() - items['pA']['task'] = ' ' - self.fails(items, "pA: placeholder or empty title ' '") - items = self.two() - items['pA']['task'] = 'no marker' - self.fails(items, "pA: title does not carry a run marker: 'no marker'") - items = self.two() - items['pA']['delegatedToId'] = 'cA' - self.fails(items, 'pA: parent still waits on a child (status \'active\')') - items = self.two() - items['pA']['awaitingChildId'] = 'cA' - self.fails(items, 'pA: parent still waits on a child (status \'active\')') - items = self.two() - items['cA']['workspace'] = 'other' - self.fails(items, 'pA: parent and child workspaces differ') - - def test_shared_workspace_and_extra_child_fail(self): - items = {**good_pair('A', 'w'), **good_pair('B', 'w')} - self.assertTrue(any('distinct workspaces' in f for f in check_items(items, 2))) - items = self.two() - items.update(good_pair('A', 'wa')) - items['cA2'] = item('cA2', 'HISTORY_RACE_CHILD A again', parentTaskId='pA', - status='completed', workspace='wa') - self.fails(items, 'pA: expected one child for tag A, found 2') - - def test_missing_task_fails(self): - items = self.two() - del items['cB'] - self.assertTrue(any('expected 2 parents' in f for f in check_items(items, 2))) - - -class StoreAndLogTest(unittest.TestCase): - def test_index_file_and_bad_json_fail(self): - with tempfile.TemporaryDirectory() as directory: - tasks = Path(directory) / 'ext' / 'tasks' - (tasks / 'x').mkdir(parents=True) - (tasks / '_index.json').write_text('{}') - (tasks / 'x' / 'history_item.json').write_text('{broken') - items, failures = load_items(Path(directory), 0) - self.assertEqual({}, items) - self.assertEqual(2, len(failures)) - - def test_id_mismatch_and_non_object_fail(self): - with tempfile.TemporaryDirectory() as directory: - tasks = Path(directory) / 'ext' / 'tasks' - for name, text in (('x', '{"id": "y"}'), ('z', '[1]')): - (tasks / name).mkdir(parents=True) - (tasks / name / 'history_item.json').write_text(text) - items, failures = load_items(Path(directory), 0) - self.assertEqual({}, items) - self.assertEqual(["x: id field is 'y'", 'z: history_item.json is not an object'], failures) - - def test_old_tasks_are_ignored(self): - with tempfile.TemporaryDirectory() as directory: - task = Path(directory) / 'ext' / 'tasks' / 'x' - task.mkdir(parents=True) - (task / 'history_item.json').write_text('{broken') - self.assertEqual(({}, []), load_items(Path(directory), 2 ** 40)) - - def test_provider_and_log_checks(self): - with tempfile.TemporaryDirectory() as directory: - provider = Path(directory) / 'p.jsonl' - good = ['barrier-released'] * 2 + ['completed'] * 6 - provider.write_text('\n'.join(json.dumps({'event': e}) for e in good)) - self.assertEqual([], check_provider(provider, 2)) - provider.write_text(json.dumps({'event': 'barrier-timeout'})) - self.assertTrue(check_provider(provider, 2)) - def provider_failures(events): - provider.write_text('\n'.join(json.dumps({'event': e}) for e in events)) - return check_provider(provider, 2) - self.assertIn('provider logged disconnected', provider_failures(good + ['disconnected'])) - self.assertIn('barrier released 1 times, expected 2', - provider_failures(['barrier-released'] + ['completed'] * 6)) - self.assertIn('5 completed streams, expected 6', - provider_failures(['barrier-released'] * 2 + ['completed'] * 5)) - self.assertIn('7 completed streams, expected 6', provider_failures(good + ['completed'])) - self.assertIn('barrier released 3 times, expected 2', provider_failures(good + ['barrier-released'])) - self.assertEqual(['provider logged barrier-timeout'], provider_failures(good + ['barrier-timeout'])) - provider.write_text('{"event": "completed"}\n{broken') - self.assertTrue(any('not valid JSON' in f for f in check_provider(provider, 2))) - provider.write_text('[1]\n' + '\n'.join(json.dumps({'event': e}) for e in good)) - self.assertEqual(["p.jsonl: provider log line is not an object: [1]"], check_provider(provider, 2)) - missing = Path(directory) / 'missing.log' - self.assertEqual([f'IDE log does not exist: {missing}'], check_logs([missing])) - log = Path(directory) / 'idea.log' - log.write_text('ok\nInvalid status transition interrupted -> delegated\n') - self.assertEqual(1, len(check_logs([log]))) - for phrase in ('invalid status transition', 'DeltaRejectedError', 'failed to persist', 'Cannot re-delegate'): - log.write_text(f'ok\n{phrase}\n') - self.assertEqual([f'idea.log:2: {phrase}'], check_logs([log]), phrase) - - -if __name__ == '__main__': - unittest.main() diff --git a/jetbrains_plugin/scripts/ui-harness/persist/test_solheim_proxy.py b/jetbrains_plugin/scripts/ui-harness/persist/test_solheim_proxy.py deleted file mode 100644 index 54e857a..0000000 --- a/jetbrains_plugin/scripts/ui-harness/persist/test_solheim_proxy.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Local relay access checks; never contact the provider.""" -import http.client -import http.server -import threading -import urllib.error -import unittest -from unittest.mock import patch -import solheim_proxy as relay - - -class RelayAccessTest(unittest.TestCase): - def setUp(self): - relay.TOKEN = 'a' * 64 - self.server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), relay.Relay) - self.worker = threading.Thread(target=self.server.serve_forever) - self.worker.start() - - def tearDown(self): - self.server.shutdown() - self.server.server_close() - self.worker.join() - - def request(self, headers, body=None, method='GET'): - connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) - with patch.object(relay.UPSTREAM, 'open') as upstream: - connection.request(method, '/v1/models', body=body, headers=headers) - response = connection.getresponse() - response.read() - connection.close() - upstream.assert_not_called() - return response.status - - def test_local_health_is_authenticated_and_does_not_contact_upstream(self): - for token, expected in [('wrong', 401), (relay.TOKEN, 200)]: - connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) - with patch.object(relay.UPSTREAM, 'open') as upstream: - connection.request('GET', '/v1/health', headers={'Authorization': 'Bearer ' + token}) - response = connection.getresponse() - self.assertEqual(expected, response.status) - response.read() - upstream.assert_not_called() - connection.close() - - def test_missing_token(self): - self.assertEqual(401, self.request({})) - - def test_wrong_token(self): - self.assertEqual(401, self.request({'Authorization': 'Bearer wrong'})) - - def test_browser_origin(self): - self.assertEqual(403, self.request({'Authorization': 'Bearer ' + relay.TOKEN, - 'Origin': 'https://example.com'})) - - def test_untrusted_host(self): - self.assertEqual(403, self.request({'Authorization': 'Bearer ' + relay.TOKEN, - 'Host': 'example.com'})) - - def test_invalid_content_length(self): - for length in ['-1', 'not-a-number', '1.5']: - with self.subTest(length=length): - self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN, - 'Content-Length': length}, method='POST')) - - def test_excessive_content_length(self): - for length in [str(relay.MAX_BODY_BYTES + 1), '9' * 5000]: - with self.subTest(length=length[:20]): - self.assertEqual(413, self.request({'Authorization': 'Bearer ' + relay.TOKEN, - 'Content-Length': length}, method='POST')) - - def test_malformed_json(self): - for body in [b'{', b'\xff']: - with self.subTest(body=body): - self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN}, - body=body, method='POST')) - - def test_non_object_json(self): - for body in [b'[]', b'null', b'1']: - with self.subTest(body=body): - self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN}, - body=body, method='POST')) - - def test_wrong_message_shape(self): - self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN}, - body=b'{"messages":null}', method='POST')) - - def test_unsupported_transfer_encoding(self): - self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN, - 'Transfer-Encoding': 'chunked'}, method='POST')) - - def test_authorized_request_forwards_only_upstream_key(self): - relay.KEY = 'private-upstream-test-key' - response = unittest.mock.Mock(status=200, headers={'Content-Type': 'application/json'}) - response.read1.side_effect = [b'{"data":[]}', b''] - connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) - with patch.object(relay.UPSTREAM, 'open', return_value=response) as upstream, patch.object(relay, 'record'): - connection.request('GET', '/v1/models', headers={'Authorization': 'Bearer ' + relay.TOKEN}) - result = connection.getresponse() - self.assertEqual(200, result.status) - self.assertEqual(b'{"data":[]}', result.read()) - self.assertEqual('Bearer private-upstream-test-key', upstream.call_args.args[0].get_header('Authorization')) - connection.close() - - def test_authorized_post_forwards_json_unchanged(self): - relay.KEY = 'private-upstream-test-key' - body = b'{"model":"test-model","messages":[],"stream":true}' - response = unittest.mock.Mock(status=200, headers={'Content-Type': 'text/event-stream'}) - response.read1.side_effect = [b'data: [DONE]\n', b''] - connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) - with patch.object(relay.UPSTREAM, 'open', return_value=response) as upstream, patch.object(relay, 'record'): - connection.request('POST', '/v1/chat/completions', body=body, - headers={'Authorization': 'Bearer ' + relay.TOKEN}) - result = connection.getresponse() - self.assertEqual(200, result.status) - self.assertEqual(b'data: [DONE]\n', result.read()) - self.assertEqual(body, upstream.call_args.args[0].data) - self.assertEqual('POST', upstream.call_args.args[0].get_method()) - connection.close() - - -class RedirectTest(unittest.TestCase): - def test_redirects_never_reach_destination(self): - class RedirectServer(http.server.BaseHTTPRequestHandler): - paths = [] - - def log_message(self, *_): - pass - - def do_GET(self): - self.paths.append(self.path) - self.send_response(int(self.path[1:])) - self.send_header('Location', '/destination') - self.send_header('Content-Length', '0') - self.end_headers() - - server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), RedirectServer) - worker = threading.Thread(target=server.serve_forever) - worker.start() - try: - for status in [301, 302, 303, 307, 308]: - with self.subTest(status=status): - request = relay.urllib.request.Request( - 'http://127.0.0.1:' + str(server.server_port) + '/' + str(status), - headers={'Authorization': 'Bearer test-only-key'}) - with self.assertRaises(urllib.error.HTTPError) as caught: - relay.UPSTREAM.open(request, timeout=5) - self.assertEqual(status, caught.exception.code) - caught.exception.close() - self.assertEqual(['/301', '/302', '/303', '/307', '/308'], RedirectServer.paths) - finally: - server.shutdown() - server.server_close() - worker.join() - - -if __name__ == '__main__': - unittest.main() diff --git a/jetbrains_plugin/scripts/ui-harness/persist/test_streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/persist/test_streaming_fixture.py deleted file mode 100644 index 72f5a85..0000000 --- a/jetbrains_plugin/scripts/ui-harness/persist/test_streaming_fixture.py +++ /dev/null @@ -1,188 +0,0 @@ -import http.client -import json -import tempfile -import threading -import unittest -from unittest.mock import patch -from pathlib import Path - -from streaming_fixture import history_role, make_server - - -class StreamingFixtureTest(unittest.TestCase): - def setUp(self): - self.directory = tempfile.TemporaryDirectory() - self.log = Path(self.directory.name) / 'provider.jsonl' - self.server = make_server(0, self.log, chunks=3, interval=0.03) - self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) - self.thread.start() - self.connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port, timeout=5) - - def tearDown(self): - self.connection.close() - self.server.shutdown() - self.server.server_close() - self.thread.join() - self.directory.cleanup() - - def post(self, body, token='focus-fixture-local-only'): - self.connection.request('POST', '/v1/chat/completions', json.dumps(body), - {'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json'}) - return self.connection.getresponse() - - def test_two_incremental_streams_finish_with_native_completion(self): - request = {'stream': True, 'messages': [], 'tools': [ - {'type': 'function', 'function': {'name': 'attempt_completion'}}]} - for turn in (1, 2): - paused = threading.Event() - release = threading.Event() - - def pause_stream(_interval): - paused.set() - if not release.wait(5): - raise TimeoutError('Test did not release the fixture stream') - - # Hold the server after its first content chunk. Receiving both the - # role and content before release proves incremental delivery without - # depending on sleep durations or the host's clock resolution. - with patch('streaming_fixture.time.sleep', side_effect=pause_stream): - try: - response = self.post(request) - self.assertEqual(200, response.status) - self.assertEqual('text/event-stream', response.getheader('Content-Type')) - first = json.loads(response.readline().decode()[6:]) - self.assertEqual('assistant', first['choices'][0]['delta']['role']) - self.assertEqual(b'\n', response.readline()) - update = json.loads(response.readline().decode()[6:]) - self.assertEqual(f'History update 1 for turn {turn}. ', - update['choices'][0]['delta']['content']) - self.assertTrue(paused.wait(5), 'Server did not pause during the stream') - finally: - # Also unblock the request handler if any assertion fails. - release.set() - events = [update] + [json.loads(line[6:]) for line in response.read().decode().splitlines() - if line.startswith('data: {')] - updates = [e for e in events if e['choices'][0]['delta'].get('content')] - self.assertEqual(3, len(updates)) - tool = events[-2]['choices'][0]['delta']['tool_calls'][0] - self.assertEqual('attempt_completion', tool['function']['name']) - self.assertIn(str(turn), json.loads(tool['function']['arguments'])['result']) - self.assertEqual('tool_calls', events[-1]['choices'][0]['finish_reason']) - records = [json.loads(line) for line in self.log.read_text().splitlines()] - self.assertEqual(2, len([e for e in records if e['event'] == 'completed'])) - - def test_rejects_non_streaming_and_missing_completion_tool(self): - for request in ({'stream': False}, {'stream': True, 'tools': []}): - response = self.post(request) - self.assertEqual(400, response.status) - response.read() - self.assertEqual(0, self.server.requests) - - def test_rejects_invalid_local_token(self): - response = self.post({'stream': True}, token='wrong') - self.assertEqual(401, response.status) - response.read() - self.assertEqual(0, self.server.requests) - - def test_health_and_model_discovery(self): - self.connection.request('GET', '/health') - response = self.connection.getresponse() - self.assertEqual(b'ready', response.read()) - self.connection.request('GET', '/v1/models') - response = self.connection.getresponse() - self.assertEqual('focus-fixture', json.loads(response.read())['data'][0]['id']) - - -class HistoryRaceScenarioTest(unittest.TestCase): - def user(self, text): - return {'role': 'user', 'content': [{'type': 'text', 'text': text}]} - - def test_roles_follow_the_first_user_message(self): - self.assertEqual(('delegate', 'A'), history_role([self.user('HISTORY_RACE_PARENT A go')])) - self.assertEqual(('child', 'B'), history_role([self.user('HISTORY_RACE_CHILD B go')])) - self.assertEqual(('plain', ''), history_role([self.user('hello')])) - - def test_parent_resumes_after_new_task_but_ignores_the_system_prompt(self): - system = {'role': 'system', 'content': 'tools: new_task, attempt_completion'} - first = self.user('HISTORY_RACE_PARENT A go') - self.assertEqual(('delegate', 'A'), history_role([system, first])) - called = {'role': 'assistant', 'tool_calls': [{'function': {'name': 'new_task'}}]} - self.assertEqual(('resume', 'A'), history_role([system, first, called])) - - def test_two_parents_delegate_together_and_children_complete(self): - directory = tempfile.TemporaryDirectory() - self.addCleanup(directory.cleanup) - log = Path(directory.name) / 'provider.jsonl' - server = make_server(0, log, chunks=2, interval=0.01, scenario='history-race', barrier=2) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - self.addCleanup(thread.join) - self.addCleanup(server.server_close) - self.addCleanup(server.shutdown) - tools = [{'type': 'function', 'function': {'name': 'attempt_completion'}}] - results = {} - - def call(tag, text): - connection = http.client.HTTPConnection('127.0.0.1', server.server_port, timeout=10) - body = {'stream': True, 'tools': tools, 'messages': [self.user(text)]} - connection.request('POST', '/v1/chat/completions', json.dumps(body), - {'Authorization': 'Bearer focus-fixture-local-only'}) - data = connection.getresponse().read().decode() - results[tag] = [json.loads(line[6:]) for line in data.splitlines() if line.startswith('data: {')] - connection.close() - - def events(): - return [json.loads(line)['event'] for line in log.read_text().splitlines()] if log.exists() else [] - - parents = {tag: threading.Thread(target=call, args=(tag, f'HISTORY_RACE_PARENT {tag}')) for tag in 'AB'} - # Parent A alone must stay blocked: no new_task and no barrier-released. - parents['A'].start() - parents['A'].join(0.5) - self.assertTrue(parents['A'].is_alive()) - self.assertNotIn('A', results) - self.assertNotIn('barrier-released', events()) - parents['B'].start() - for parent in parents.values(): - parent.join(15) - self.assertFalse(parent.is_alive()) - for tag in 'AB': - call_delta = results[tag][-2]['choices'][0]['delta']['tool_calls'][0]['function'] - self.assertEqual('new_task', call_delta['name']) - arguments = json.loads(call_delta['arguments']) - self.assertEqual(f'HISTORY_RACE_CHILD {tag} reply with attempt_completion.', arguments['message']) - self.assertIsNone(arguments['todos']) - call('A', 'HISTORY_RACE_CHILD A go') - done = results['A'][-2]['choices'][0]['delta']['tool_calls'][0]['function']['name'] - self.assertEqual('attempt_completion', done) - events = [json.loads(line)['event'] for line in log.read_text().splitlines()] - self.assertEqual(2, events.count('barrier-released')) - self.assertNotIn('barrier-timeout', events) - counts = [json.loads(line)['chunks'] for line in log.read_text().splitlines() - if json.loads(line)['event'] == 'completed'] - self.assertEqual([2, 3, 3], sorted(counts)) - - def test_lone_parent_times_out_instead_of_passing(self): - directory = tempfile.TemporaryDirectory() - self.addCleanup(directory.cleanup) - log = Path(directory.name) / 'provider.jsonl' - server = make_server(0, log, chunks=1, interval=0.01, scenario='history-race', - barrier=2, barrier_timeout=0.3) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - self.addCleanup(thread.join) - self.addCleanup(server.server_close) - self.addCleanup(server.shutdown) - connection = http.client.HTTPConnection('127.0.0.1', server.server_port, timeout=10) - body = {'stream': True, 'tools': [{'type': 'function', 'function': {'name': 'attempt_completion'}}], - 'messages': [self.user('HISTORY_RACE_PARENT A go')]} - connection.request('POST', '/v1/chat/completions', json.dumps(body), - {'Authorization': 'Bearer focus-fixture-local-only'}) - connection.getresponse().read() - connection.close() - events = [json.loads(line)['event'] for line in log.read_text().splitlines()] - self.assertIn('barrier-timeout', events) - self.assertNotIn('barrier-released', events) - - -if __name__ == '__main__': - unittest.main() diff --git a/jetbrains_plugin/scripts/ui-harness/re-delegate/check_history_store.py b/jetbrains_plugin/scripts/ui-harness/re-delegate/check_history_store.py deleted file mode 100644 index b76de99..0000000 --- a/jetbrains_plugin/scripts/ui-harness/re-delegate/check_history_store.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python3 -"""Check the shared Zoo Code task history after a concurrent two-IDE run. - -Issue #6: several IDE processes write one global history store. A safe store -keeps one valid history_item.json per task, with no shared index file and with -parent and child links that stay inside one IDE's task tree. -""" -import argparse -import json -import re -import sys -from pathlib import Path - -PLACEHOLDER = re.compile(r'^\s*(Task|Work) #\d+') -BAD_LOG = re.compile(r'invalid status transition|DeltaRejectedError|failed to persist|Cannot re-delegate', re.I) -MARKER = re.compile(r'HISTORY_RACE_(PARENT|CHILD) (\w+)') - - -def load_items(storage, since): - """Return ({task id: item}, failures) for tasks written at or after `since`.""" - items, failures = {}, [] - for index in storage.glob('**/tasks/_index.json'): - if index.stat().st_mtime >= since: - failures.append(f'shared index file was written: {index}') - for path in sorted(storage.glob('**/tasks/*/history_item.json')): - if path.stat().st_mtime < since: - continue - try: - item = json.loads(path.read_text(encoding='utf-8')) - except (OSError, ValueError) as error: - failures.append(f'{path.parent.name}: history_item.json is not valid JSON ({error})') - continue - if not isinstance(item, dict): - failures.append(f'{path.parent.name}: history_item.json is not an object') - continue - if item.get('id') != path.parent.name: - failures.append(f'{path.parent.name}: id field is {item.get("id")!r}') - continue - items[item['id']] = item - return items, failures - - -def check_items(items, instances): - failures = [] - tasks = {} - for task_id, item in items.items(): - text = item.get('task') or '' - if PLACEHOLDER.match(text) or not text.strip(): - failures.append(f'{task_id}: placeholder or empty title {text!r}') - continue - match = MARKER.search(text) - if not match: - failures.append(f'{task_id}: title does not carry a run marker: {text!r}') - continue - if not item.get('size'): - failures.append(f'{task_id}: size is {item.get("size")!r} after real work') - tasks[task_id] = (match.group(1), match.group(2), item) - parents = {i: t for i, t in tasks.items() if t[0] == 'PARENT'} - children = {i: t for i, t in tasks.items() if t[0] == 'CHILD'} - if len(parents) != instances or len(children) != instances: - failures.append(f'expected {instances} parents and {instances} children, ' - f'found {len(parents)} and {len(children)}') - for parent_id, (_, tag, parent) in parents.items(): - own = [c for c, t in children.items() if t[1] == tag] - if len(own) != 1: - failures.append(f'{parent_id}: expected one child for tag {tag}, found {len(own)}') - continue - child_id = own[0] - child = children[child_id][2] - if parent.get('childIds') != [child_id]: - failures.append(f'{parent_id}: childIds {parent.get("childIds")!r}, expected {[child_id]!r}') - if child.get('parentTaskId') != parent_id: - failures.append(f'{child_id}: parentTaskId {child.get("parentTaskId")!r}, expected {parent_id!r}') - if child.get('status') != 'completed': - failures.append(f'{child_id}: child status {child.get("status")!r}, expected completed') - if parent.get('status') == 'delegated' or parent.get('awaitingChildId') or parent.get('delegatedToId'): - failures.append(f'{parent_id}: parent still waits on a child (status {parent.get("status")!r})') - if parent.get('workspace') != child.get('workspace'): - failures.append(f'{parent_id}: parent and child workspaces differ') - workspaces = {t[2].get('workspace') for t in parents.values()} - if instances > 1 and len(workspaces) != instances: - failures.append(f'expected {instances} distinct workspaces, found {sorted(map(str, workspaces))}') - return failures - - -def check_provider(path, instances): - events, failures = [], [] - for number, line in enumerate(path.read_text(encoding='utf-8').splitlines(), 1): - if not line.strip(): - continue - try: - events.append(json.loads(line)) - except ValueError as error: - failures.append(f'{path.name}:{number}: provider log line is not valid JSON ({error})') - for event in events: - if not isinstance(event, dict): - failures.append(f'{path.name}: provider log line is not an object: {event!r}') - names = [e.get('event') for e in events if isinstance(e, dict)] - if names.count('barrier-released') != instances: - failures.append(f'barrier released {names.count("barrier-released")} times, expected {instances}') - for bad in ('barrier-timeout', 'disconnected'): - if bad in names: - failures.append(f'provider logged {bad}') - if names.count('completed') != 3 * instances: - failures.append(f'{names.count("completed")} completed streams, expected {3 * instances}') - return failures - - -def check_logs(paths): - failures = [] - for path in paths: - if not path.exists(): - failures.append(f'IDE log does not exist: {path}') - continue - for number, line in enumerate(path.read_text(encoding='utf-8', errors='replace').splitlines(), 1): - if BAD_LOG.search(line): - failures.append(f'{path.name}:{number}: {line.strip()[:200]}') - return failures - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--storage', type=Path, required=True) - parser.add_argument('--since', type=float, required=True, help='epoch seconds when the run started') - parser.add_argument('--provider-log', type=Path, required=True) - parser.add_argument('--ide-log', type=Path, action='append', required=True) - parser.add_argument('--instances', type=int, default=2) - parser.add_argument('--report', type=Path) - args = parser.parse_args() - items, failures = load_items(args.storage, args.since) - failures += check_items(items, args.instances) - failures += check_provider(args.provider_log, args.instances) - failures += check_logs(args.ide_log) - report = {'tasks': len(items), 'failures': failures} - text = json.dumps(report, indent=2) - if args.report: - args.report.write_text(text, encoding='utf-8') - print(text) - return 1 if failures else 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/jetbrains_plugin/scripts/ui-harness/re-delegate/solheim_proxy.py b/jetbrains_plugin/scripts/ui-harness/re-delegate/solheim_proxy.py deleted file mode 100644 index 669102d..0000000 --- a/jetbrains_plugin/scripts/ui-harness/re-delegate/solheim_proxy.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python3 -"""Loopback authentication relay for manual Solheim extension focus smoke tests. - -Real credentials stay in SOLHEIM_KEY_FILE (mode 0600), outside IDE settings/logs. -The extension uses a per-run relay token and http://127.0.0.1:19487/v1. Logs contain only -request metadata, upstream status, stream byte counts, and tool names. -""" -import http.server -import hmac -import json -import os -import time -import urllib.error -import urllib.request -from pathlib import Path - -KEY = None -TOKEN = None -BASE = 'https://api.solheim.ai/v1' -MAX_BODY_BYTES = 8 * 1024 * 1024 -LOG = Path(os.environ.get('SOLHEIM_PROXY_LOG', '/tmp/zoo-solheim-proxy.jsonl')) - - -def record(**fields): - with LOG.open('a') as output: - output.write(json.dumps({'time': time.time(), **fields}) + '\n') - - -class RefuseRedirects(urllib.request.HTTPRedirectHandler): - def redirect_request(self, request, response, code, message, headers, new_url): - # Never send provider credentials to a redirected URL. - return None - - -UPSTREAM = urllib.request.build_opener(RefuseRedirects()) - - -class Relay(http.server.BaseHTTPRequestHandler): - protocol_version = 'HTTP/1.1' - - def log_message(self, *_): - pass - - def do_GET(self): - self.relay() - - def do_POST(self): - self.relay() - - def relay(self): - expected_host = '127.0.0.1:' + str(self.server.server_port) - if self.headers.get('Host') != expected_host or self.headers.get('Origin') is not None: - self.send_error(403) - return - supplied = self.headers.get('Authorization', '') - if not TOKEN or not hmac.compare_digest(supplied, 'Bearer ' + TOKEN): - self.send_error(401) - return - if not self.path.startswith('/v1/'): - self.send_error(404) - return - if self.command == 'GET' and self.path == '/v1/health': - # Local readiness only; never probe the paid upstream in a startup loop. - self.send_response(200) - self.send_header('Content-Length', '5') - self.end_headers() - self.wfile.write(b'ready') - return - lengths = self.headers.get_all('Content-Length', []) - length_text = lengths[0] if lengths else '0' - if (len(lengths) > 1 or self.headers.get('Transfer-Encoding') is not None - or not length_text.isascii() or not length_text.isdecimal()): - self.send_error(400, 'Invalid request framing') - return - # Check before int conversion, including absurdly long decimal strings. - if len(length_text.lstrip('0')) > 8 or int(length_text.lstrip('0') or '0') > MAX_BODY_BYTES: - self.send_error(413, 'Request body too large') - return - length = int(length_text.lstrip('0') or '0') - body = self.rfile.read(length) - try: - parsed = json.loads(body) if body else {} - if len(body) != length or not isinstance(parsed, dict): - raise ValueError('Expected a JSON object') - if not isinstance(parsed.get('messages', []), list): - raise ValueError('Expected a message list') - except (ValueError, UnicodeError): - self.send_error(400, 'Invalid JSON request body') - return - record(event='request', path=self.path, model=parsed.get('model'), - messages=len(parsed.get('messages', [])), stream=parsed.get('stream', False)) - request = urllib.request.Request(BASE + self.path[3:], data=body or None, - method=self.command, - headers={'Authorization': 'Bearer ' + KEY, - 'Content-Type': 'application/json'}) - try: - response = UPSTREAM.open(request, timeout=180) - except urllib.error.HTTPError as error: - response = error - except Exception as error: - record(event='transport-error', error=type(error).__name__) - self.send_error(502) - return - self.send_response(response.status) - self.send_header('Content-Type', response.headers.get('Content-Type', 'application/json')) - self.send_header('Connection', 'close') - self.end_headers() - count = 0 - buffer = b'' - tools = set() - try: - while True: - chunk = response.read1(4096) - if not chunk: - break - count += len(chunk) - self.wfile.write(chunk) - self.wfile.flush() - buffer += chunk - while b'\n' in buffer: - line, buffer = buffer.split(b'\n', 1) - if line.startswith(b'data: ') and line != b'data: [DONE]': - try: - event = json.loads(line[6:]) - for choice in event.get('choices', []): - for tool in choice.get('delta', {}).get('tool_calls', []): - name = tool.get('function', {}).get('name') - if name: - tools.add(name) - except (ValueError, TypeError): - pass - except (BrokenPipeError, ConnectionResetError): - record(event='client-disconnected') - finally: - response.close() - self.close_connection = True - record(event='response', status=response.status, bytes=count, tools=sorted(tools)) - - -if __name__ == '__main__': - KEY = Path(os.environ['SOLHEIM_KEY_FILE']).read_text().strip() - TOKEN = os.environ['SOLHEIM_RELAY_TOKEN'] - if len(TOKEN) < 32: - raise SystemExit('Use a random relay token of at least 32 characters') - http.server.ThreadingHTTPServer(('127.0.0.1', 19487), Relay).serve_forever() diff --git a/jetbrains_plugin/scripts/ui-harness/re-delegate/streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/re-delegate/streaming_fixture.py deleted file mode 100644 index 07d7af8..0000000 --- a/jetbrains_plugin/scripts/ui-harness/re-delegate/streaming_fixture.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic loopback OpenAI fixture for headful chat-focus CI tests. - -Only the provider is simulated. The extension, React history, JCEF, and native -keyboard delivery remain real. No commands or file edits are requested. -""" -import argparse -import http.server -import json -import re -import threading -import time -from pathlib import Path - - -def _text(content): - if isinstance(content, list): - return ''.join(part.get('text', '') for part in content if isinstance(part, dict)) - return content if isinstance(content, str) else '' - - -def history_role(messages): - """Classify a request in the history-race scenario. - - The first user message names the task: a parent carries HISTORY_RACE_PARENT - and a child carries HISTORY_RACE_CHILD . A parent that already holds a - new_task call in a non-system message has delegated, so it resumes. - """ - users = [m for m in messages if m.get('role') == 'user'] - first = _text(users[0].get('content')) if users else '' - child = re.search(r'HISTORY_RACE_CHILD (\w+)', first) - if child: - return 'child', child.group(1) - parent = re.search(r'HISTORY_RACE_PARENT (\w+)', first) - if not parent: - return 'plain', '' - body = json.dumps([m for m in messages if m.get('role') != 'system']) - return ('resume' if 'new_task' in body else 'delegate'), parent.group(1) - - -class Fixture(http.server.BaseHTTPRequestHandler): - def log_message(self, *_): - pass - - def do_GET(self): - if self.path == '/health': - self.send_response(200) - self.end_headers() - self.wfile.write(b'ready') - elif self.path == '/v1/models': - self.send_response(200) - self.send_header('Content-Type', 'application/json') - self.end_headers() - self.wfile.write(json.dumps({'object': 'list', 'data': [ - {'id': 'focus-fixture', 'object': 'model', 'owned_by': 'fixture'} - ]}).encode()) - else: - self.send_error(404) - - def do_POST(self): - if self.path != '/v1/chat/completions': - self.send_error(404) - return - if self.headers.get('Authorization') != 'Bearer focus-fixture-local-only': - self.send_error(401) - return - try: - length = int(self.headers.get('Content-Length', '0')) - if not 0 < length <= 8 * 1024 * 1024: - raise ValueError('body size') - request = json.loads(self.rfile.read(length)) - if request.get('stream') is not True: - raise ValueError('streaming required') - tools = [t.get('function', {}).get('name') for t in request.get('tools', [])] - if 'attempt_completion' not in tools: - raise ValueError('native attempt_completion tool required') - except (ValueError, TypeError, AttributeError): - self.send_error(400, 'Expected streaming request with native completion tool') - return - with self.server.counter_lock: - self.server.requests += 1 - turn = self.server.requests - role, tag = 'plain', '' - if self.server.scenario == 'history-race': - role, tag = history_role(request.get('messages', [])) - self.server.record(event='request', turn=turn, tools=tools, role=role, tag=tag) - self.send_response(200) - self.send_header('Content-Type', 'text/event-stream') - self.send_header('Cache-Control', 'no-cache') - self.end_headers() - - def emit(delta, finish=None): - chunk = {'id': f'focus-{turn}', 'object': 'chat.completion.chunk', - 'created': int(time.time()), 'model': 'focus-fixture', - 'choices': [{'index': 0, 'delta': delta, 'finish_reason': finish}]} - self.wfile.write(('data: ' + json.dumps(chunk) + '\n\n').encode()) - self.wfile.flush() - - try: - emit({'role': 'assistant', 'content': ''}) - chunks = self.server.chunks - if role == 'delegate': - # Hold every parent until all IDEs have submitted, so the delegations - # and the child streams that follow overlap in time. - try: - self.server.barrier.wait() - except threading.BrokenBarrierError: - self.server.record(event='barrier-timeout', turn=turn, tag=tag) - return - self.server.record(event='barrier-released', turn=turn, tag=tag) - if role in ('delegate', 'resume'): - chunks = 3 - for index in range(chunks): - emit({'content': f'History update {index + 1} for turn {turn}. '}) - time.sleep(self.server.interval) - if role == 'delegate': - call = ('new_task', {'mode': 'code', 'todos': None, - 'message': f'HISTORY_RACE_CHILD {tag} reply with attempt_completion.'}) - else: - call = ('attempt_completion', {'result': f'Fixture turn {turn} complete.'}) - emit({'tool_calls': [{'index': 0, 'id': f'call-{turn}', - 'type': 'function', 'function': { - 'name': call[0], 'arguments': json.dumps(call[1])}}]}) - emit({}, 'tool_calls') - self.wfile.write(b'data: [DONE]\n\n') - self.wfile.flush() - self.server.record(event='completed', turn=turn, chunks=chunks) - except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError): - self.server.record(event='disconnected', turn=turn) - - -def make_server(port=19487, log=None, chunks=100, interval=0.25, scenario=None, barrier=2, barrier_timeout=300): - server = http.server.ThreadingHTTPServer(('127.0.0.1', port), Fixture) - server.requests = 0 - server.counter_lock = threading.Lock() - server.chunks, server.interval = chunks, interval - server.scenario = scenario - server.barrier = threading.Barrier(barrier, timeout=barrier_timeout) - - def record(**fields): - if log: - with server.counter_lock, Path(log).open('a', encoding='utf-8') as output: - output.write(json.dumps({'time': time.time(), **fields}) + '\n') - server.record = record - return server - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--port', type=int, default=19487) - parser.add_argument('--log', type=Path, required=True) - parser.add_argument('--scenario', choices=['history-race']) - parser.add_argument('--chunks', type=int, default=100) - parser.add_argument('--interval', type=float, default=0.25) - parser.add_argument('--barrier', type=int, default=2) - args = parser.parse_args() - make_server(args.port, args.log, args.chunks, args.interval, args.scenario, args.barrier).serve_forever() diff --git a/jetbrains_plugin/scripts/ui-harness/re-delegate/test_check_history_store.py b/jetbrains_plugin/scripts/ui-harness/re-delegate/test_check_history_store.py deleted file mode 100644 index b095026..0000000 --- a/jetbrains_plugin/scripts/ui-harness/re-delegate/test_check_history_store.py +++ /dev/null @@ -1,235 +0,0 @@ -import json -import tempfile -import unittest -from pathlib import Path - -from check_history_store import check_items, check_logs, check_provider, load_items - - -def item(task_id, text, **extra): - return {'id': task_id, 'task': text, 'size': 10, 'workspace': 'w', **extra} - - -def good_pair(tag, workspace): - return { - 'p' + tag: item('p' + tag, f'HISTORY_RACE_PARENT {tag} go', childIds=['c' + tag], - status='active', workspace=workspace), - 'c' + tag: item('c' + tag, f'HISTORY_RACE_CHILD {tag} go', parentTaskId='p' + tag, - status='completed', workspace=workspace), - } - - -# Real items from Windows CI run 43562237505 (commit 5a494cf, IDEA 2026.2.1, extension 3.96.0). -# Only the temporary run folder name in `workspace` was shortened. -REAL_ITEMS = json.loads(r'''[ - { - "id": "01a1d898-c7ab-70da-936e-6467341e0125", - "parentTaskId": "01a1d898-bf28-71a1-92d9-df474bf72871", - "number": 1, - "ts": 1794635304779, - "task": "HISTORY_RACE_CHILD A reply with attempt_completion.", - "tokensIn": 0, - "tokensOut": 0, - "totalCost": 0, - "size": 10621, - "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-A", - "mode": "code", - "apiConfigName": "focus-fixture", - "status": "completed", - "completionResultSummary": "Fixture turn 4 complete." - }, - { - "id": "01a1d898-c663-72e9-b6dd-1caba4bda93a", - "parentTaskId": "01a1d898-4435-75fd-afc2-6ca3aa4c89a4", - "number": 1, - "ts": 1794635304331, - "task": "HISTORY_RACE_CHILD B reply with attempt_completion.", - "tokensIn": 0, - "tokensOut": 0, - "totalCost": 0, - "size": 10623, - "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-B", - "mode": "code", - "apiConfigName": "focus-fixture", - "status": "completed", - "completionResultSummary": "Fixture turn 3 complete." - }, - { - "id": "01a1d898-bf28-71a1-92d9-df474bf72871", - "number": 1, - "ts": 1794635306508, - "task": "HISTORY_RACE_PARENT A delegate one subtask, then finish.", - "tokensIn": 0, - "tokensOut": 0, - "totalCost": 0, - "size": 7694, - "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-A", - "mode": "code", - "apiConfigName": "focus-fixture", - "status": "active", - "childIds": [ - "01a1d898-c7ab-70da-936e-6467341e0125" - ], - "completedByChildId": "01a1d898-c7ab-70da-936e-6467341e0125", - "completionResultSummary": "Fixture turn 4 complete." - }, - { - "id": "01a1d898-4435-75fd-afc2-6ca3aa4c89a4", - "number": 1, - "ts": 1794635306029, - "task": "HISTORY_RACE_PARENT B delegate one subtask, then finish.", - "tokensIn": 0, - "tokensOut": 0, - "totalCost": 0, - "size": 7698, - "workspace": "c:\\Users\\runneradmin\\AppData\\Local\\Temp\\zoo-focus-RUN\\workspace-B", - "mode": "code", - "apiConfigName": "focus-fixture", - "status": "active", - "childIds": [ - "01a1d898-c663-72e9-b6dd-1caba4bda93a" - ], - "completedByChildId": "01a1d898-c663-72e9-b6dd-1caba4bda93a", - "completionResultSummary": "Fixture turn 3 complete." - } -]''') - - -class CheckItemsTest(unittest.TestCase): - def test_real_extension_items_pass(self): - items = {i['id']: i for i in REAL_ITEMS} - self.assertEqual(4, len(items)) - self.assertEqual([], check_items(items, 2)) - - - def two(self): - return {**good_pair('A', 'wa'), **good_pair('B', 'wb')} - - def test_clean_store_passes(self): - self.assertEqual([], check_items(self.two(), 2)) - - def test_placeholder_title_and_zero_size_fail(self): - items = self.two() - items['pA']['task'] = 'Task #1' - items['cB']['size'] = 0 - failures = check_items(items, 2) - self.assertTrue(any('placeholder' in f for f in failures)) - self.assertTrue(any('size is 0' in f for f in failures)) - - def test_cross_linked_children_fail(self): - items = self.two() - items['pA']['childIds'] = ['cB'] - items['cA']['parentTaskId'] = 'pB' - failures = check_items(items, 2) - self.assertTrue(any('childIds' in f for f in failures)) - self.assertTrue(any('parentTaskId' in f for f in failures)) - - def test_stuck_delegation_and_open_child_fail(self): - items = self.two() - items['pA'].update(status='delegated', awaitingChildId='cA') - items['cA']['status'] = 'interrupted' - failures = check_items(items, 2) - self.assertTrue(any('still waits' in f for f in failures)) - self.assertTrue(any('child status' in f for f in failures)) - - def fails(self, items, message): - self.assertIn(message, check_items(items, 2)) - - def test_each_bad_item_branch_reports_an_exact_message(self): - items = self.two() - items['pA']['task'] = 'Work #2' - self.fails(items, "pA: placeholder or empty title 'Work #2'") - items = self.two() - items['pA']['task'] = ' ' - self.fails(items, "pA: placeholder or empty title ' '") - items = self.two() - items['pA']['task'] = 'no marker' - self.fails(items, "pA: title does not carry a run marker: 'no marker'") - items = self.two() - items['pA']['delegatedToId'] = 'cA' - self.fails(items, 'pA: parent still waits on a child (status \'active\')') - items = self.two() - items['pA']['awaitingChildId'] = 'cA' - self.fails(items, 'pA: parent still waits on a child (status \'active\')') - items = self.two() - items['cA']['workspace'] = 'other' - self.fails(items, 'pA: parent and child workspaces differ') - - def test_shared_workspace_and_extra_child_fail(self): - items = {**good_pair('A', 'w'), **good_pair('B', 'w')} - self.assertTrue(any('distinct workspaces' in f for f in check_items(items, 2))) - items = self.two() - items.update(good_pair('A', 'wa')) - items['cA2'] = item('cA2', 'HISTORY_RACE_CHILD A again', parentTaskId='pA', - status='completed', workspace='wa') - self.fails(items, 'pA: expected one child for tag A, found 2') - - def test_missing_task_fails(self): - items = self.two() - del items['cB'] - self.assertTrue(any('expected 2 parents' in f for f in check_items(items, 2))) - - -class StoreAndLogTest(unittest.TestCase): - def test_index_file_and_bad_json_fail(self): - with tempfile.TemporaryDirectory() as directory: - tasks = Path(directory) / 'ext' / 'tasks' - (tasks / 'x').mkdir(parents=True) - (tasks / '_index.json').write_text('{}') - (tasks / 'x' / 'history_item.json').write_text('{broken') - items, failures = load_items(Path(directory), 0) - self.assertEqual({}, items) - self.assertEqual(2, len(failures)) - - def test_id_mismatch_and_non_object_fail(self): - with tempfile.TemporaryDirectory() as directory: - tasks = Path(directory) / 'ext' / 'tasks' - for name, text in (('x', '{"id": "y"}'), ('z', '[1]')): - (tasks / name).mkdir(parents=True) - (tasks / name / 'history_item.json').write_text(text) - items, failures = load_items(Path(directory), 0) - self.assertEqual({}, items) - self.assertEqual(["x: id field is 'y'", 'z: history_item.json is not an object'], failures) - - def test_old_tasks_are_ignored(self): - with tempfile.TemporaryDirectory() as directory: - task = Path(directory) / 'ext' / 'tasks' / 'x' - task.mkdir(parents=True) - (task / 'history_item.json').write_text('{broken') - self.assertEqual(({}, []), load_items(Path(directory), 2 ** 40)) - - def test_provider_and_log_checks(self): - with tempfile.TemporaryDirectory() as directory: - provider = Path(directory) / 'p.jsonl' - good = ['barrier-released'] * 2 + ['completed'] * 6 - provider.write_text('\n'.join(json.dumps({'event': e}) for e in good)) - self.assertEqual([], check_provider(provider, 2)) - provider.write_text(json.dumps({'event': 'barrier-timeout'})) - self.assertTrue(check_provider(provider, 2)) - def provider_failures(events): - provider.write_text('\n'.join(json.dumps({'event': e}) for e in events)) - return check_provider(provider, 2) - self.assertIn('provider logged disconnected', provider_failures(good + ['disconnected'])) - self.assertIn('barrier released 1 times, expected 2', - provider_failures(['barrier-released'] + ['completed'] * 6)) - self.assertIn('5 completed streams, expected 6', - provider_failures(['barrier-released'] * 2 + ['completed'] * 5)) - self.assertIn('7 completed streams, expected 6', provider_failures(good + ['completed'])) - self.assertIn('barrier released 3 times, expected 2', provider_failures(good + ['barrier-released'])) - self.assertEqual(['provider logged barrier-timeout'], provider_failures(good + ['barrier-timeout'])) - provider.write_text('{"event": "completed"}\n{broken') - self.assertTrue(any('not valid JSON' in f for f in check_provider(provider, 2))) - provider.write_text('[1]\n' + '\n'.join(json.dumps({'event': e}) for e in good)) - self.assertEqual(["p.jsonl: provider log line is not an object: [1]"], check_provider(provider, 2)) - missing = Path(directory) / 'missing.log' - self.assertEqual([f'IDE log does not exist: {missing}'], check_logs([missing])) - log = Path(directory) / 'idea.log' - log.write_text('ok\nInvalid status transition interrupted -> delegated\n') - self.assertEqual(1, len(check_logs([log]))) - for phrase in ('invalid status transition', 'DeltaRejectedError', 'failed to persist', 'Cannot re-delegate'): - log.write_text(f'ok\n{phrase}\n') - self.assertEqual([f'idea.log:2: {phrase}'], check_logs([log]), phrase) - - -if __name__ == '__main__': - unittest.main() diff --git a/jetbrains_plugin/scripts/ui-harness/re-delegate/test_solheim_proxy.py b/jetbrains_plugin/scripts/ui-harness/re-delegate/test_solheim_proxy.py deleted file mode 100644 index 54e857a..0000000 --- a/jetbrains_plugin/scripts/ui-harness/re-delegate/test_solheim_proxy.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Local relay access checks; never contact the provider.""" -import http.client -import http.server -import threading -import urllib.error -import unittest -from unittest.mock import patch -import solheim_proxy as relay - - -class RelayAccessTest(unittest.TestCase): - def setUp(self): - relay.TOKEN = 'a' * 64 - self.server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), relay.Relay) - self.worker = threading.Thread(target=self.server.serve_forever) - self.worker.start() - - def tearDown(self): - self.server.shutdown() - self.server.server_close() - self.worker.join() - - def request(self, headers, body=None, method='GET'): - connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) - with patch.object(relay.UPSTREAM, 'open') as upstream: - connection.request(method, '/v1/models', body=body, headers=headers) - response = connection.getresponse() - response.read() - connection.close() - upstream.assert_not_called() - return response.status - - def test_local_health_is_authenticated_and_does_not_contact_upstream(self): - for token, expected in [('wrong', 401), (relay.TOKEN, 200)]: - connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) - with patch.object(relay.UPSTREAM, 'open') as upstream: - connection.request('GET', '/v1/health', headers={'Authorization': 'Bearer ' + token}) - response = connection.getresponse() - self.assertEqual(expected, response.status) - response.read() - upstream.assert_not_called() - connection.close() - - def test_missing_token(self): - self.assertEqual(401, self.request({})) - - def test_wrong_token(self): - self.assertEqual(401, self.request({'Authorization': 'Bearer wrong'})) - - def test_browser_origin(self): - self.assertEqual(403, self.request({'Authorization': 'Bearer ' + relay.TOKEN, - 'Origin': 'https://example.com'})) - - def test_untrusted_host(self): - self.assertEqual(403, self.request({'Authorization': 'Bearer ' + relay.TOKEN, - 'Host': 'example.com'})) - - def test_invalid_content_length(self): - for length in ['-1', 'not-a-number', '1.5']: - with self.subTest(length=length): - self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN, - 'Content-Length': length}, method='POST')) - - def test_excessive_content_length(self): - for length in [str(relay.MAX_BODY_BYTES + 1), '9' * 5000]: - with self.subTest(length=length[:20]): - self.assertEqual(413, self.request({'Authorization': 'Bearer ' + relay.TOKEN, - 'Content-Length': length}, method='POST')) - - def test_malformed_json(self): - for body in [b'{', b'\xff']: - with self.subTest(body=body): - self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN}, - body=body, method='POST')) - - def test_non_object_json(self): - for body in [b'[]', b'null', b'1']: - with self.subTest(body=body): - self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN}, - body=body, method='POST')) - - def test_wrong_message_shape(self): - self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN}, - body=b'{"messages":null}', method='POST')) - - def test_unsupported_transfer_encoding(self): - self.assertEqual(400, self.request({'Authorization': 'Bearer ' + relay.TOKEN, - 'Transfer-Encoding': 'chunked'}, method='POST')) - - def test_authorized_request_forwards_only_upstream_key(self): - relay.KEY = 'private-upstream-test-key' - response = unittest.mock.Mock(status=200, headers={'Content-Type': 'application/json'}) - response.read1.side_effect = [b'{"data":[]}', b''] - connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) - with patch.object(relay.UPSTREAM, 'open', return_value=response) as upstream, patch.object(relay, 'record'): - connection.request('GET', '/v1/models', headers={'Authorization': 'Bearer ' + relay.TOKEN}) - result = connection.getresponse() - self.assertEqual(200, result.status) - self.assertEqual(b'{"data":[]}', result.read()) - self.assertEqual('Bearer private-upstream-test-key', upstream.call_args.args[0].get_header('Authorization')) - connection.close() - - def test_authorized_post_forwards_json_unchanged(self): - relay.KEY = 'private-upstream-test-key' - body = b'{"model":"test-model","messages":[],"stream":true}' - response = unittest.mock.Mock(status=200, headers={'Content-Type': 'text/event-stream'}) - response.read1.side_effect = [b'data: [DONE]\n', b''] - connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port) - with patch.object(relay.UPSTREAM, 'open', return_value=response) as upstream, patch.object(relay, 'record'): - connection.request('POST', '/v1/chat/completions', body=body, - headers={'Authorization': 'Bearer ' + relay.TOKEN}) - result = connection.getresponse() - self.assertEqual(200, result.status) - self.assertEqual(b'data: [DONE]\n', result.read()) - self.assertEqual(body, upstream.call_args.args[0].data) - self.assertEqual('POST', upstream.call_args.args[0].get_method()) - connection.close() - - -class RedirectTest(unittest.TestCase): - def test_redirects_never_reach_destination(self): - class RedirectServer(http.server.BaseHTTPRequestHandler): - paths = [] - - def log_message(self, *_): - pass - - def do_GET(self): - self.paths.append(self.path) - self.send_response(int(self.path[1:])) - self.send_header('Location', '/destination') - self.send_header('Content-Length', '0') - self.end_headers() - - server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), RedirectServer) - worker = threading.Thread(target=server.serve_forever) - worker.start() - try: - for status in [301, 302, 303, 307, 308]: - with self.subTest(status=status): - request = relay.urllib.request.Request( - 'http://127.0.0.1:' + str(server.server_port) + '/' + str(status), - headers={'Authorization': 'Bearer test-only-key'}) - with self.assertRaises(urllib.error.HTTPError) as caught: - relay.UPSTREAM.open(request, timeout=5) - self.assertEqual(status, caught.exception.code) - caught.exception.close() - self.assertEqual(['/301', '/302', '/303', '/307', '/308'], RedirectServer.paths) - finally: - server.shutdown() - server.server_close() - worker.join() - - -if __name__ == '__main__': - unittest.main() diff --git a/jetbrains_plugin/scripts/ui-harness/re-delegate/test_streaming_fixture.py b/jetbrains_plugin/scripts/ui-harness/re-delegate/test_streaming_fixture.py deleted file mode 100644 index 72f5a85..0000000 --- a/jetbrains_plugin/scripts/ui-harness/re-delegate/test_streaming_fixture.py +++ /dev/null @@ -1,188 +0,0 @@ -import http.client -import json -import tempfile -import threading -import unittest -from unittest.mock import patch -from pathlib import Path - -from streaming_fixture import history_role, make_server - - -class StreamingFixtureTest(unittest.TestCase): - def setUp(self): - self.directory = tempfile.TemporaryDirectory() - self.log = Path(self.directory.name) / 'provider.jsonl' - self.server = make_server(0, self.log, chunks=3, interval=0.03) - self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) - self.thread.start() - self.connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port, timeout=5) - - def tearDown(self): - self.connection.close() - self.server.shutdown() - self.server.server_close() - self.thread.join() - self.directory.cleanup() - - def post(self, body, token='focus-fixture-local-only'): - self.connection.request('POST', '/v1/chat/completions', json.dumps(body), - {'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json'}) - return self.connection.getresponse() - - def test_two_incremental_streams_finish_with_native_completion(self): - request = {'stream': True, 'messages': [], 'tools': [ - {'type': 'function', 'function': {'name': 'attempt_completion'}}]} - for turn in (1, 2): - paused = threading.Event() - release = threading.Event() - - def pause_stream(_interval): - paused.set() - if not release.wait(5): - raise TimeoutError('Test did not release the fixture stream') - - # Hold the server after its first content chunk. Receiving both the - # role and content before release proves incremental delivery without - # depending on sleep durations or the host's clock resolution. - with patch('streaming_fixture.time.sleep', side_effect=pause_stream): - try: - response = self.post(request) - self.assertEqual(200, response.status) - self.assertEqual('text/event-stream', response.getheader('Content-Type')) - first = json.loads(response.readline().decode()[6:]) - self.assertEqual('assistant', first['choices'][0]['delta']['role']) - self.assertEqual(b'\n', response.readline()) - update = json.loads(response.readline().decode()[6:]) - self.assertEqual(f'History update 1 for turn {turn}. ', - update['choices'][0]['delta']['content']) - self.assertTrue(paused.wait(5), 'Server did not pause during the stream') - finally: - # Also unblock the request handler if any assertion fails. - release.set() - events = [update] + [json.loads(line[6:]) for line in response.read().decode().splitlines() - if line.startswith('data: {')] - updates = [e for e in events if e['choices'][0]['delta'].get('content')] - self.assertEqual(3, len(updates)) - tool = events[-2]['choices'][0]['delta']['tool_calls'][0] - self.assertEqual('attempt_completion', tool['function']['name']) - self.assertIn(str(turn), json.loads(tool['function']['arguments'])['result']) - self.assertEqual('tool_calls', events[-1]['choices'][0]['finish_reason']) - records = [json.loads(line) for line in self.log.read_text().splitlines()] - self.assertEqual(2, len([e for e in records if e['event'] == 'completed'])) - - def test_rejects_non_streaming_and_missing_completion_tool(self): - for request in ({'stream': False}, {'stream': True, 'tools': []}): - response = self.post(request) - self.assertEqual(400, response.status) - response.read() - self.assertEqual(0, self.server.requests) - - def test_rejects_invalid_local_token(self): - response = self.post({'stream': True}, token='wrong') - self.assertEqual(401, response.status) - response.read() - self.assertEqual(0, self.server.requests) - - def test_health_and_model_discovery(self): - self.connection.request('GET', '/health') - response = self.connection.getresponse() - self.assertEqual(b'ready', response.read()) - self.connection.request('GET', '/v1/models') - response = self.connection.getresponse() - self.assertEqual('focus-fixture', json.loads(response.read())['data'][0]['id']) - - -class HistoryRaceScenarioTest(unittest.TestCase): - def user(self, text): - return {'role': 'user', 'content': [{'type': 'text', 'text': text}]} - - def test_roles_follow_the_first_user_message(self): - self.assertEqual(('delegate', 'A'), history_role([self.user('HISTORY_RACE_PARENT A go')])) - self.assertEqual(('child', 'B'), history_role([self.user('HISTORY_RACE_CHILD B go')])) - self.assertEqual(('plain', ''), history_role([self.user('hello')])) - - def test_parent_resumes_after_new_task_but_ignores_the_system_prompt(self): - system = {'role': 'system', 'content': 'tools: new_task, attempt_completion'} - first = self.user('HISTORY_RACE_PARENT A go') - self.assertEqual(('delegate', 'A'), history_role([system, first])) - called = {'role': 'assistant', 'tool_calls': [{'function': {'name': 'new_task'}}]} - self.assertEqual(('resume', 'A'), history_role([system, first, called])) - - def test_two_parents_delegate_together_and_children_complete(self): - directory = tempfile.TemporaryDirectory() - self.addCleanup(directory.cleanup) - log = Path(directory.name) / 'provider.jsonl' - server = make_server(0, log, chunks=2, interval=0.01, scenario='history-race', barrier=2) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - self.addCleanup(thread.join) - self.addCleanup(server.server_close) - self.addCleanup(server.shutdown) - tools = [{'type': 'function', 'function': {'name': 'attempt_completion'}}] - results = {} - - def call(tag, text): - connection = http.client.HTTPConnection('127.0.0.1', server.server_port, timeout=10) - body = {'stream': True, 'tools': tools, 'messages': [self.user(text)]} - connection.request('POST', '/v1/chat/completions', json.dumps(body), - {'Authorization': 'Bearer focus-fixture-local-only'}) - data = connection.getresponse().read().decode() - results[tag] = [json.loads(line[6:]) for line in data.splitlines() if line.startswith('data: {')] - connection.close() - - def events(): - return [json.loads(line)['event'] for line in log.read_text().splitlines()] if log.exists() else [] - - parents = {tag: threading.Thread(target=call, args=(tag, f'HISTORY_RACE_PARENT {tag}')) for tag in 'AB'} - # Parent A alone must stay blocked: no new_task and no barrier-released. - parents['A'].start() - parents['A'].join(0.5) - self.assertTrue(parents['A'].is_alive()) - self.assertNotIn('A', results) - self.assertNotIn('barrier-released', events()) - parents['B'].start() - for parent in parents.values(): - parent.join(15) - self.assertFalse(parent.is_alive()) - for tag in 'AB': - call_delta = results[tag][-2]['choices'][0]['delta']['tool_calls'][0]['function'] - self.assertEqual('new_task', call_delta['name']) - arguments = json.loads(call_delta['arguments']) - self.assertEqual(f'HISTORY_RACE_CHILD {tag} reply with attempt_completion.', arguments['message']) - self.assertIsNone(arguments['todos']) - call('A', 'HISTORY_RACE_CHILD A go') - done = results['A'][-2]['choices'][0]['delta']['tool_calls'][0]['function']['name'] - self.assertEqual('attempt_completion', done) - events = [json.loads(line)['event'] for line in log.read_text().splitlines()] - self.assertEqual(2, events.count('barrier-released')) - self.assertNotIn('barrier-timeout', events) - counts = [json.loads(line)['chunks'] for line in log.read_text().splitlines() - if json.loads(line)['event'] == 'completed'] - self.assertEqual([2, 3, 3], sorted(counts)) - - def test_lone_parent_times_out_instead_of_passing(self): - directory = tempfile.TemporaryDirectory() - self.addCleanup(directory.cleanup) - log = Path(directory.name) / 'provider.jsonl' - server = make_server(0, log, chunks=1, interval=0.01, scenario='history-race', - barrier=2, barrier_timeout=0.3) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - self.addCleanup(thread.join) - self.addCleanup(server.server_close) - self.addCleanup(server.shutdown) - connection = http.client.HTTPConnection('127.0.0.1', server.server_port, timeout=10) - body = {'stream': True, 'tools': [{'type': 'function', 'function': {'name': 'attempt_completion'}}], - 'messages': [self.user('HISTORY_RACE_PARENT A go')]} - connection.request('POST', '/v1/chat/completions', json.dumps(body), - {'Authorization': 'Bearer focus-fixture-local-only'}) - connection.getresponse().read() - connection.close() - events = [json.loads(line)['event'] for line in log.read_text().splitlines()] - self.assertIn('barrier-timeout', events) - self.assertNotIn('barrier-released', events) - - -if __name__ == '__main__': - unittest.main() diff --git a/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py b/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py index b095026..65da66b 100644 --- a/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py +++ b/jetbrains_plugin/scripts/ui-harness/test_check_history_store.py @@ -124,6 +124,12 @@ def test_cross_linked_children_fail(self): self.assertTrue(any('childIds' in f for f in failures)) self.assertTrue(any('parentTaskId' in f for f in failures)) + def test_foreign_or_missing_child_ids_fail(self): + for child_ids in (['cA', 'cB'], []): + items = self.two() + items['pA']['childIds'] = child_ids + self.assertTrue(any('childIds' in f for f in check_items(items, 2)), child_ids) + def test_stuck_delegation_and_open_child_fail(self): items = self.two() items['pA'].update(status='delegated', awaitingChildId='cA') @@ -229,6 +235,9 @@ def provider_failures(events): for phrase in ('invalid status transition', 'DeltaRejectedError', 'failed to persist', 'Cannot re-delegate'): log.write_text(f'ok\n{phrase}\n') self.assertEqual([f'idea.log:2: {phrase}'], check_logs([log]), phrase) + stamped = '2026-01-01 12:00:00,000 [ 1] WARN - Invalid status transition x' + log.write_text(f'ok\n{stamped}\n') + self.assertEqual([f'idea.log:2: {stamped}'], check_logs([log])) if __name__ == '__main__':