From c539a3014f34931f536d67eb37e4e5581d7bd1db Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:10:50 +0000 Subject: [PATCH 01/29] vendors: stop relabelling grok/kimi PostCompact as canonical pre_compact Finding raw[14].findings[12] of the vendor-truth review, reproduced by execution: both entries mapped the vendor's post-compaction event onto pre_compact beside their real PreCompact, so a handler that exists to snapshot context before compaction also fired after it had already happened. Devin's identical defect was closed in PR #43; the "unknown" treatment Kimi's four aliases got in PR #59 keeps claims() identifying the payload, so only the relabelling stops. wire_events already pinned pre_compact to PreCompact on both, so install output is unchanged. Signed-off-by: Claude Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 11 +++++++++++ src/agentseam/data/vendors/grok.json | 2 +- src/agentseam/data/vendors/kimi_code.json | 2 +- tests/test_adapter_grok.py | 14 ++++++++++++++ tests/test_adapter_kimi_code.py | 10 ++++++++++ 5 files changed, 37 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebbb463..340e49d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,17 @@ versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html). re-run -- still show their original drift. ### Fixed +- **`PostCompact` no longer masquerades as canonical `pre_compact` on Grok and Kimi Code** + (`data/vendors/grok.json`, `data/vendors/kimi_code.json`; vendor-truth review finding + `raw[14].findings[12]`). Both entries mapped the vendor's + post-compaction event onto `pre_compact` alongside their real `PreCompact`, so a handler + written to snapshot context *before* compaction discards it also fired *after* it had + already happened, with no way to tell the two moments apart except by reading + `event.raw`. Devin's identical defect was closed this way in PR #43; the same + `"unknown"` treatment Kimi's four aliases got in PR #59 is used here, so `claims()` + still identifies the payload and only the relabelling stops. `REVERSE_EVENT_MAP` and + therefore what `install` writes are unchanged -- `wire_events` already pinned + `pre_compact` to `PreCompact` on both. - **The stop gate's block observable is the hook re-firing, not a second action run** (`tools/experiment.py`). `_blocked()` used to score a Stop-gate block by reading the sentinel twice, on the theory that an agent refused permission to finish comes back diff --git a/src/agentseam/data/vendors/grok.json b/src/agentseam/data/vendors/grok.json index 6347b5d..26fc12a 100644 --- a/src/agentseam/data/vendors/grok.json +++ b/src/agentseam/data/vendors/grok.json @@ -11,7 +11,7 @@ "display": "Grok CLI", "events": { "PermissionDenied": "tool_failure", - "PostCompact": "pre_compact", + "PostCompact": "unknown", "PostToolUse": "post_tool", "PostToolUseFailure": "tool_failure", "PreCompact": "pre_compact", diff --git a/src/agentseam/data/vendors/kimi_code.json b/src/agentseam/data/vendors/kimi_code.json index 40de71e..9826345 100644 --- a/src/agentseam/data/vendors/kimi_code.json +++ b/src/agentseam/data/vendors/kimi_code.json @@ -16,7 +16,7 @@ "Interrupt": "unknown", "PermissionRequest": "unknown", "PermissionResult": "post_tool", - "PostCompact": "pre_compact", + "PostCompact": "unknown", "PostToolUse": "post_tool", "PostToolUseFailure": "tool_failure", "PreCompact": "pre_compact", diff --git a/tests/test_adapter_grok.py b/tests/test_adapter_grok.py index 0a7d78b..39287c1 100644 --- a/tests/test_adapter_grok.py +++ b/tests/test_adapter_grok.py @@ -74,3 +74,17 @@ def test_hook_config_uses_claude_codes_shape_in_groks_own_file(): def test_project_hooks_are_recorded_as_needing_trust(): """A written config is not a running one: Grok gates project hooks behind /hooks-trust.""" assert A.adapters.get("grok").NEEDS_TRUST is True + + +def test_post_compact_is_not_bent_into_pre_compact(): + """PostCompact fires AFTER compaction; a pre_compact policy exists to run BEFORE it.""" + mod = A.adapters.get("grok") + assert mod.parse({"hookEventName": "PostCompact", "sessionId": "s"}).event == A.UNKNOWN + assert mod.parse({"hookEventName": "PreCompact", "sessionId": "s"}).event == A.PRE_COMPACT + assert mod.REVERSE_EVENT_MAP[A.PRE_COMPACT] == "PreCompact" + + +def test_a_post_compact_payload_is_still_claimed_so_the_moment_is_not_lost(): + """Unmapping relabels the event; it must not blind a caller to the payload.""" + mod = A.adapters.get("grok") + assert mod.claims({"hookEventName": "PostCompact", "sessionId": "s"}) diff --git a/tests/test_adapter_kimi_code.py b/tests/test_adapter_kimi_code.py index bf7eb85..ecfd71f 100644 --- a/tests/test_adapter_kimi_code.py +++ b/tests/test_adapter_kimi_code.py @@ -131,3 +131,13 @@ def test_kimi_blocks_but_fails_open_and_the_notes_say_not_to_rely_on_it(): assert not A.can_rewrite("kimi_code", A.PRE_TOOL) assert A.enforcement_level("kimi_code", A.PRE_TOOL) == "best-effort" assert "not a sole security barrier" in A.MATRIX["kimi_code"]["notes"] + + +def test_post_compact_is_not_bent_into_pre_compact(): + """PostCompact fires AFTER compaction; a pre_compact policy exists to run BEFORE it.""" + mod = A.adapters.get("kimi_code") + post = {"hook_event_name": "PostCompact", "client_type": "kimi_code_cli"} + assert mod.parse(post).event == A.UNKNOWN + assert mod.parse({"hook_event_name": "PreCompact", "client_type": "kimi_code_cli"}).event == A.PRE_COMPACT + assert mod.REVERSE_EVENT_MAP[A.PRE_COMPACT] == "PreCompact" + assert mod.claims(post), "unmapping must not blind a caller to the payload" From 7e608ecb32468f9e89c63a2bee7397c81adcd624 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:04:31 +0000 Subject: [PATCH 02/29] evidence_report: a report records which canonical event it measured experiment_report.as_report() now reads `event` off the trial results (every result already carries one) instead of leaving it implicit in the file the report happens to be pasted under. evidence_report.py accepts `event` as an optional field, validated against contract.EVENTS when present, and carries it through to_evidence()/diff_against() so a maintainer merging a submission can see which gate it measured rather than assuming the row's own. Existing reports with no `event` stay valid. The evidence-report.yml issue template gains a matching optional field. Co-Authored-By: Claude Sonnet 5 Signed-off-by: Claude --- .github/ISSUE_TEMPLATE/evidence-report.yml | 9 +++++++++ src/agentseam/evidence_report.py | 7 ++++++- tests/test_evidence_report.py | 19 +++++++++++++++++++ tools/experiment_report.py | 5 ++++- 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/evidence-report.yml b/.github/ISSUE_TEMPLATE/evidence-report.yml index e651890..093e556 100644 --- a/.github/ISSUE_TEMPLATE/evidence-report.yml +++ b/.github/ISSUE_TEMPLATE/evidence-report.yml @@ -54,6 +54,15 @@ body: placeholder: Windows 11 / macOS 15 / Ubuntu 24.04 validations: required: true + - type: input + id: event + attributes: + label: Gate + description: > + Which canonical event this report measured (prompt_submit, pre_tool, stop, ...). + Matches `--report`'s own `event` field when you used `tools/experiment.py`; leave + blank for a `capture.py` payload-shape report, which is not scoped to one gate. + placeholder: pre_tool - type: textarea id: report attributes: diff --git a/src/agentseam/evidence_report.py b/src/agentseam/evidence_report.py index 803f191..d588b07 100644 --- a/src/agentseam/evidence_report.py +++ b/src/agentseam/evidence_report.py @@ -23,6 +23,7 @@ from __future__ import annotations +from .contract import EVENTS from .matrix_terms import BASES, BASIS_DOCS, BASIS_LIVE, BASIS_LIVE_PARTIAL #: Current report format. Bumped when a required field changes, so an old submission is @@ -44,6 +45,7 @@ "run_url", "notes", "tool_version", + "event", ) #: The driver name the harness uses for its credential-free documentation simulator. @@ -109,6 +111,9 @@ def validate(report): "watched cannot be checked for drift later." % report["basis"] ) + if report.get("event") is not None and report["event"] not in EVENTS: + raise InvalidReportError("event %r is not one of: %s" % (report["event"], ", ".join(EVENTS))) + if not str(report["date"]).count("-") == 2: # noqa: PLR2004 raise InvalidReportError("date must be YYYY-MM-DD, got %r" % (report["date"],)) @@ -128,7 +133,7 @@ def to_evidence(report): "version": report.get("version") or "unrecorded", "method": report.get("notes") or "submitted evidence report", } - for key in ("observed", "experiments", "reporter", "run_url"): + for key in ("observed", "experiments", "reporter", "run_url", "event"): if report.get(key): entry[key] = report[key] return entry diff --git a/tests/test_evidence_report.py b/tests/test_evidence_report.py index 2005f60..e29d0c7 100644 --- a/tests/test_evidence_report.py +++ b/tests/test_evidence_report.py @@ -119,3 +119,22 @@ def test_diff_writes_nothing(): before = dict(EVIDENCE["cursor"]) er.diff_against(EVIDENCE["cursor"], _report()) assert EVIDENCE["cursor"] == before + + +def test_a_report_round_trips_with_its_event(): + row = er.to_evidence(_report(event="pre_tool")) + assert row["event"] == "pre_tool" + delta = er.diff_against(EVIDENCE["cursor"], _report(event="pre_tool")) + assert delta["changes"]["event"] == (None, "pre_tool") + + +def test_a_report_round_trips_without_an_event(): + """Existing reports carry no `event`; absence must stay valid (task 1, W55).""" + assert er.validate(_report()) + row = er.to_evidence(_report()) + assert "event" not in row + + +def test_a_bad_event_value_is_rejected_by_name(): + with pytest.raises(er.InvalidReportError, match="event"): + er.validate(_report(event="mid_tool")) diff --git a/tools/experiment_report.py b/tools/experiment_report.py index c565f8f..d4325ee 100644 --- a/tools/experiment_report.py +++ b/tools/experiment_report.py @@ -63,7 +63,9 @@ def as_report(results, *, version=None, reporter=None, notes=None, today=None): The basis is derived from the driver, never chosen by the caller: a run against the reference is documentation and says so. evidence_report.validate() enforces the same - rule independently, so a hand-edited report cannot claim more than it earned. + rule independently, so a hand-edited report cannot claim more than it earned. `event` + is likewise read off the trials rather than passed in, so a report says which gate it + measured and cannot be merged into a claim about a different one by accident. """ driver = results[0]["driver"] measured = {} @@ -77,6 +79,7 @@ def as_report(results, *, version=None, reporter=None, notes=None, today=None): "driver": "reference" if driver == evidence_report.REFERENCE_DRIVER else "real-agent", "experiments": measured, "platform": sys.platform, + "event": results[0]["event"], } for key, value in (("version", version), ("reporter", reporter), ("notes", notes)): if value: From 5289a33237e83c27df805c49767d7178f6cd83b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:10:08 +0000 Subject: [PATCH 03/29] experiment: an escalate trial, answered in the agent's own dialect Add trial `escalate` (tools/experiment_probe.py): the probe answers with Decision.escalate() rendered through adapter.respond(), so Cursor's reply spells it "ask" and an agent whose gate does not honour escalate gets its own degraded-block dialect, exactly like every other trial's answer. The new measured field `escalate_means` needs a third value neither `silence_means` nor `unknown_verb_means` has to express: `prompted`, when the run ended waiting on an answer nobody gave, distinct from a definite `refusal-or-error`. The sentinel alone cannot tell those apart -- both leave it untouched -- so classification also reads the driver's own outcome (non-zero exit or timeout, sentinel untouched, reads as `prompted`); split into tools/experiment_escalate.py, which carries the exact rule in its docstring, to keep experiment.py under the line budget. tools/experiment.py's own real-driver invocation moves to tools/experiment_driver.py for the same reason, gaining a caught TimeoutExpired so a stalled headless run is observed rather than propagated as a crash. tools/reference_agent.py now raises Undocumented for PreToolUse's `permissionDecision: "ask"`: the value is documented, but what a headless run does next with nobody there to answer is not, so a guess here would launder that silence into evidence -- the same discipline the unknown-verb trial already gets. escalate_means joins matrix_terms.OPTIONAL_CLAIM_FIELDS with the same "absence is not a claim" rule as W53's fields. Co-Authored-By: Claude Sonnet 5 Signed-off-by: Claude --- src/agentseam/matrix_terms.py | 14 ++- tests/test_experiment_kit_escalate.py | 121 ++++++++++++++++++++++++++ tools/experiment.py | 52 +++++------ tools/experiment_driver.py | 44 ++++++++++ tools/experiment_escalate.py | 69 +++++++++++++++ tools/experiment_probe.py | 2 + tools/reference_agent.py | 7 +- 7 files changed, 281 insertions(+), 28 deletions(-) create mode 100644 tests/test_experiment_kit_escalate.py create mode 100644 tools/experiment_driver.py create mode 100644 tools/experiment_escalate.py diff --git a/src/agentseam/matrix_terms.py b/src/agentseam/matrix_terms.py index 9ae3c6d..c0e3ec5 100644 --- a/src/agentseam/matrix_terms.py +++ b/src/agentseam/matrix_terms.py @@ -40,12 +40,18 @@ def _cap(*, block=False, rewrite=False, fail=FAIL_OPEN): CLAIM_SILENCE_MEANS = "silence_means" CLAIM_TIMEOUT_FAIL_MODE = "timeout_fail_mode" CLAIM_UNKNOWN_VERB_MEANS = "unknown_verb_means" +CLAIM_ESCALATE_MEANS = "escalate_means" #: Always present on every cell the row claims; each needs its own evidence record. REQUIRED_CLAIM_FIELDS = (CLAIM_BLOCK, CLAIM_REWRITE, CLAIM_FAIL_MODE) #: Present only where measured (task 3, 2026-09-07): absence is not a claim, so these need #: evidence only on the cells that actually carry them. -OPTIONAL_CLAIM_FIELDS = (CLAIM_SILENCE_MEANS, CLAIM_TIMEOUT_FAIL_MODE, CLAIM_UNKNOWN_VERB_MEANS) +OPTIONAL_CLAIM_FIELDS = ( + CLAIM_SILENCE_MEANS, + CLAIM_TIMEOUT_FAIL_MODE, + CLAIM_UNKNOWN_VERB_MEANS, + CLAIM_ESCALATE_MEANS, +) CLAIM_FIELDS = REQUIRED_CLAIM_FIELDS + OPTIONAL_CLAIM_FIELDS #: The two-value vocabulary `silence_means` and `unknown_verb_means` answer in. @@ -53,6 +59,12 @@ def _cap(*, block=False, rewrite=False, fail=FAIL_OPEN): MEANS_REFUSAL_OR_ERROR = "refusal-or-error" MEANS_VALUES = (MEANS_ALLOW, MEANS_REFUSAL_OR_ERROR) +#: `escalate_means` shares that vocabulary but adds a third state neither other field can +#: express: the run ended waiting on an answer nobody gave, rather than reaching either +#: `allow` or a definite refusal (tools/experiment_escalate.py classifies it). +MEANS_PROMPTED = "prompted" +ESCALATE_MEANS_VALUES = (MEANS_PROMPTED, MEANS_ALLOW, MEANS_REFUSAL_OR_ERROR) + GRADE_ENFORCED = "enforced" GRADE_ENFORCEABLE = "enforceable" GRADE_BEST_EFFORT = "best-effort" diff --git a/tests/test_experiment_kit_escalate.py b/tests/test_experiment_kit_escalate.py new file mode 100644 index 0000000..ef4f3ea --- /dev/null +++ b/tests/test_experiment_kit_escalate.py @@ -0,0 +1,121 @@ +"""The `escalate` trial (W55): a three-value field the sentinel alone cannot classify. + +Split from test_experiment_kit.py by activity (and to stay under the line budget): those +tests exercise the harness generally, these are specific to the one trial whose answer +depends on the driver's own outcome as well as the sentinel. +""" + +from __future__ import annotations + +import os +import sys + +import pytest + +TOOLS = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "tools") +sys.path.insert(0, TOOLS) + +import experiment # noqa: E402 +import experiment_escalate # noqa: E402 +import experiment_probe # noqa: E402 +import experiment_report # noqa: E402 +import reference_agent # noqa: E402 + +AGENT = "claude_code" + + +class _Reply: + def __init__(self, stdout="", returncode=0, stderr=""): + self.stdout, self.returncode, self.stderr = stdout, returncode, stderr + + +def test_escalate_is_documented(): + assert "escalate" in experiment_probe.BEHAVIOURS + + +def test_the_reference_cannot_settle_pre_tool_escalate(): + """PreToolUse recognises `permissionDecision: "ask"` but nothing says what a headless + run does next -- the same refusal-to-guess as the unknown-verb trial (task 6, W53).""" + r = experiment.run_trial(AGENT, "escalate") + assert r["measured"] == {"escalate_means": "undocumented"} + assert "headless run" in r["reading"] + reply = _Reply('{"hookSpecificOutput": {"permissionDecision": "ask"}}') + with pytest.raises(reference_agent.Undocumented): + reference_agent._interpret(reply, "G2") + + +def test_a_gate_that_does_not_honour_escalate_degrades_to_a_real_block(): + """Stop's gate never sees the ambiguous "ask" at all: it degrades to a block, which the + reference reads exactly as it reads any other refusal.""" + r = experiment.run_trial(AGENT, "escalate", event="stop") + assert r["measured"] == {"escalate_means": "refusal-or-error"} + + +def test_the_field_is_recognized_by_the_diff_and_starts_unasserted(): + from agentseam import matrix_terms + + assert matrix_terms.CLAIM_ESCALATE_MEANS in matrix_terms.CLAIM_FIELDS + r = experiment.run_trial(AGENT, "escalate") + rows = experiment_report.diff_against_matrix([r]) + assert rows[0]["field"] == "escalate_means" + assert rows[0]["status"] == "unasserted" + assert rows[0]["asserted"] is None + + +def test_cursor_renders_escalate_as_ask(): + """P7 (org-plan plan/agentseam-project.md): the ask row, in Cursor's own dialect.""" + from agentseam import adapters, contract + + adapter = adapters.get("cursor") + raw = { + "hook_event_name": "beforeShellExecution", + "command": "echo hi", + "cwd": ".", + "conversation_id": "c1", + } + event = adapter.parse(raw) + text, code = adapter.respond(contract.Decision.escalate("need confirmation"), event) + assert code == 0 + assert '"ask"' in text + + +def test_an_agent_that_cannot_escalate_degrades_to_its_own_block_dialect(): + from agentseam import adapters, contract + + adapter = adapters.get("codex_cli") + raw = {"hook_event_name": "PreToolUse", "tool_name": "Bash", "tool_input": {"command": "x"}} + event = adapter.parse(raw) + text, code = adapter.respond(contract.Decision.escalate("need confirmation"), event) + assert code == 0 + assert '"permissionDecision": "deny"' in text + assert "does not support ask" in text + + +# --- classify_escalate, tested directly (no live driver in CI can ever stall) ------------- + + +def test_not_blocked_reads_as_allow(): + field, value, reading = experiment_escalate.classify_escalate(False, None) + assert (field, value) == ("escalate_means", "allow") + assert "despite" in reading + + +def test_blocked_with_a_clean_driver_exit_reads_as_refusal_or_error(): + field, value, reading = experiment_escalate.classify_escalate(True, {"returncode": 0, "timed_out": False}) + assert (field, value) == ("escalate_means", "refusal-or-error") + assert "without ever running" in reading + + +def test_blocked_with_a_stalled_driver_reads_as_prompted(): + """A headless driver that hangs (or is killed) waiting for an answer nobody gave is a + different shape from a definite refusal -- the whole reason this trial exists.""" + for outcome in ({"returncode": None, "timed_out": True}, {"returncode": 1, "timed_out": False}): + field, value, reading = experiment_escalate.classify_escalate(True, outcome) + assert (field, value) == ("escalate_means", "prompted"), outcome + assert "waiting" in reading + + +def test_the_reference_driver_never_stalls(): + """Fully scripted, so its outcome can never read as a stall -- only a real driver can.""" + assert not experiment_escalate.driver_stalled({"gates": {}, "action_runs": 0, "continuations": 0}) + assert not experiment_escalate.driver_stalled(None) diff --git a/tools/experiment.py b/tools/experiment.py index 9e95d3e..6e7bda2 100644 --- a/tools/experiment.py +++ b/tools/experiment.py @@ -36,7 +36,6 @@ import json import os import shutil -import subprocess import sys import tempfile @@ -44,6 +43,8 @@ sys.path.insert(0, os.path.join(HERE, "..", "src")) sys.path.insert(0, HERE) +import experiment_driver # noqa: E402 +import experiment_escalate # noqa: E402 import experiment_probe # noqa: E402 import experiment_report # noqa: E402 import reference_agent # noqa: E402 @@ -73,8 +74,16 @@ "silence": ("silence_means", "allow", "refusal-or-error"), "timeout": ("timeout_fail_mode", FAIL_OPEN, FAIL_CLOSED), "unknown": ("unknown_verb_means", "allow", "refusal-or-error"), + # escalate is a three-value field (experiment_escalate.classify_escalate handles it + # directly in _classify below); kept here only so the table lists every trial. + "escalate": ("escalate_means", "allow", "refusal-or-error"), } +#: Trials whose Undocumented reading is named for the trial's own measured field (task 6, +#: W53) rather than the generic diagnostic "documented" -- so the diff can compare a real +#: agent's answer against the one thing the reference refuses to guess. +_UNDOCUMENTED_NAMES_ITS_FIELD = ("unknown", "escalate") + def _write_config(adapter, workspace, probe_command, event): """Wire the probe into a config *inside the scratch workspace* and return its path.""" @@ -125,11 +134,13 @@ def _invocations(record_dir): return [json.loads(line) for line in fh if line.strip()] -def _classify(trial, event, observed, invocations): +def _classify(trial, event, observed, invocations, outcome=None): """What one trial measured, as a (field, value) pair plus a human reading. `invocations` is a count, not a flag: at the stop gate how many times the hook fired is itself the measurement (see _blocked), and zero still means it never fired at all. + `outcome` is the driver's own result, needed only by the escalate trial -- see + experiment_escalate.py. """ if not invocations: return "hook_reached", False, "the hook never fired -- config path or format is wrong for this version" @@ -145,6 +156,9 @@ def _classify(trial, event, observed, invocations): return "transform", None, "ambiguous: %s" % observed blocked = _blocked(event, observed, invocations) + if trial == "escalate": + return experiment_escalate.classify_escalate(blocked, outcome) + field, when_ran, when_blocked = _MEANING[trial] if event == contract.STOP: reading = "the agent was made to continue" if blocked else "the agent finished" @@ -191,23 +205,24 @@ def run_trial(agent, trial, *, event=contract.PRE_TOOL, driver="reference", keep # here, so only a real run can settle it. undocumented, outcome = str(exc), None else: - outcome = _drive_real(driver, workspace) + outcome = experiment_driver.drive_real(driver, workspace, trigger=TRIGGER) observed = _observe(workspace) invocations = _invocations(record_dir) if undocumented is not None: - # Named for the `unknown` trial's own field (unknown_verb_means) rather than - # the generic "documented", so the diff can compare it against a real agent's - # reading of that same field -- documentation-silence vs an observed answer is - # exactly the disagreement worth surfacing. Any other trial keeps the older, - # purely diagnostic "documented" reading; nothing else currently reaches here. - if trial == "unknown": - field, value = _MEANING["unknown"][0], "undocumented" + # Named for the trial's own measured field (unknown_verb_means, escalate_means) + # rather than the generic "documented", so the diff can compare it against a + # real agent's reading of that same field -- documentation-silence vs an + # observed answer is exactly the disagreement worth surfacing. Any other trial + # keeps the older, purely diagnostic "documented" reading; nothing else + # currently reaches here. + if trial in _UNDOCUMENTED_NAMES_ITS_FIELD: + field, value = _MEANING[trial][0], "undocumented" else: field, value = "documented", False reading = "protocol is silent: %s" % undocumented else: - field, value, reading = _classify(trial, event, observed, len(invocations)) + field, value, reading = _classify(trial, event, observed, len(invocations), outcome) return { "agent": agent, "trial": trial, @@ -225,21 +240,6 @@ def run_trial(agent, trial, *, event=contract.PRE_TOOL, driver="reference", keep shutil.rmtree(workspace, ignore_errors=True) -def _drive_real(command, workspace): - """Drive a real agent CLI in the scratch workspace. - - `command` is a shell command template containing {prompt}. Kept deliberately thin: - every agent's headless invocation differs, and encoding those here would rot faster - than the matrix does. - """ - prompt = "Run this exact shell command and nothing else: %s" % TRIGGER - filled = command.replace("{prompt}", json.dumps(prompt)) - proc = subprocess.run( # noqa: S602 - filled, shell=True, cwd=workspace, capture_output=True, text=True, timeout=300 - ) - return {"returncode": proc.returncode, "stdout": proc.stdout[-2000:], "stderr": proc.stderr[-2000:]} - - def main(argv=None): parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) sub = parser.add_subparsers(dest="cmd", required=True) diff --git a/tools/experiment_driver.py b/tools/experiment_driver.py new file mode 100644 index 0000000..58e1175 --- /dev/null +++ b/tools/experiment_driver.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Driving a real agent CLI headlessly, as opposed to the reference driver's scripted replay. + +Split out of experiment.py because invoking a subprocess and interpreting a measurement are +different activities: this module knows nothing about trials, sentinels or the matrix, only +how to run one shell command and report what happened, including a stall. +""" + +from __future__ import annotations + +import json +import subprocess + + +def drive_real(command, workspace, *, trigger): + """Drive a real agent CLI in the scratch workspace. + + `command` is a shell command template containing {prompt}. Kept deliberately thin: + every agent's headless invocation differs, and encoding those here would rot faster + than the matrix does. + + A stall is not an error here: the escalate trial can legitimately leave a headless + driver waiting on an approval nobody will give, and `timed_out` is how + experiment_escalate.py tells that apart from a driver that ran and finished. + """ + prompt = "Run this exact shell command and nothing else: %s" % trigger + filled = command.replace("{prompt}", json.dumps(prompt)) + try: + proc = subprocess.run( # noqa: S602 + filled, shell=True, cwd=workspace, capture_output=True, text=True, timeout=300 + ) + except subprocess.TimeoutExpired as exc: + return { + "returncode": None, + "timed_out": True, + "stdout": (exc.stdout or "")[-2000:], + "stderr": (exc.stderr or "")[-2000:], + } + return { + "returncode": proc.returncode, + "timed_out": False, + "stdout": proc.stdout[-2000:], + "stderr": proc.stderr[-2000:], + } diff --git a/tools/experiment_escalate.py b/tools/experiment_escalate.py new file mode 100644 index 0000000..8dafb0e --- /dev/null +++ b/tools/experiment_escalate.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Classifying the `escalate` trial: a third state neither `deny` nor `silence` can reach. + +Every other trial's observable is binary -- blocked, or not, per experiment.py's own +`_blocked()` (which already knows the stop gate's asymmetry: the sentinel there fires once +on the unhooked first pass regardless of trial, so "blocked" means a *re-fire*, not "the +sentinel stayed at zero"). Escalate adds a state that binary alone cannot distinguish: a +headless driver stuck waiting on a prompt nobody is there to answer looks exactly as +"blocked" as one that was cleanly refused. Telling them apart needs the driver's own +outcome alongside that verdict. + +The exact rule, given `blocked` (experiment.py's own `_blocked()` result) and `outcome` +(the driver's result): + +* Not blocked -> ``allow``: the action happened despite being asked to defer. +* Blocked, and the driver exited non-zero or timed out -> ``prompted``: the run ended + waiting on an answer nobody gave, which is a different shape from a definite refusal. +* Blocked, and the driver exited cleanly -> ``refusal-or-error``: a decision was reached (a + block, a fail-open default, whatever the vendor does) without ever running the action -- + the same vocabulary `silence_means`/`unknown_verb_means` already read blocked as. + +The reference driver never reaches this module for the one gate where the ambiguity is +real (`PreToolUse`'s `permissionDecision: "ask"`): tools/reference_agent.py raises +`Undocumented` there instead, because nothing in Claude Code's own docs says what a +headless run does with nobody there to answer. Where the reference driver's gate degrades +`escalate` into a real block (a gate that does not honour it), it never stalls -- its +outcome carries neither `timed_out` nor a non-zero `returncode` -- so it can only ever +read `allow` or `refusal-or-error` here, never `prompted`. +""" + +from __future__ import annotations + +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..", "src")) + +from agentseam.matrix_terms import ( # noqa: E402 + CLAIM_ESCALATE_MEANS, + MEANS_ALLOW, + MEANS_PROMPTED, + MEANS_REFUSAL_OR_ERROR, +) + + +def driver_stalled(outcome): + """Whether `outcome` (a driver's own result, not the sentinel) looks like a stall. + + The reference driver's outcome (`reference_agent.run_turn`'s return value) carries + neither key and so never stalls by this reading -- it is fully scripted and always + finishes. A real headless driver's outcome (tools/experiment_driver.drive_real) carries + both. + """ + if not outcome: + return False + if outcome.get("timed_out"): + return True + returncode = outcome.get("returncode") + return returncode not in (0, None) + + +def classify_escalate(blocked, outcome): + """(field, value, reading) for the escalate trial. See the module docstring for the rule.""" + if not blocked: + return CLAIM_ESCALATE_MEANS, MEANS_ALLOW, "action ran despite being asked to defer" + if driver_stalled(outcome): + return CLAIM_ESCALATE_MEANS, MEANS_PROMPTED, "the run ended waiting on an answer nobody gave" + return CLAIM_ESCALATE_MEANS, MEANS_REFUSAL_OR_ERROR, "a decision was reached without ever running the action" diff --git a/tools/experiment_probe.py b/tools/experiment_probe.py index ac9bbfb..1f96639 100644 --- a/tools/experiment_probe.py +++ b/tools/experiment_probe.py @@ -35,6 +35,7 @@ "timeout": "Sleeps past the agent's hook timeout. Measures the fail mode under a stall.", "transform": "Rewrites the tool input. Measures whether the rewritten input is what runs.", "unknown": "Answers with a decision verb no vendor defines. Measures tolerance for the unrecognised.", + "escalate": "Defers to the host's own approval path. Measures what happens with nobody there to answer.", } #: Trials that answer nothing at all. Everything else goes through parse/respond. @@ -102,6 +103,7 @@ "deny": lambda: contract.Decision.deny("agentseam experiment: deny trial"), "transform": lambda: contract.Decision.transform( {"command": TRIGGER_ALT}, "agentseam experiment: transform trial"), + "escalate": lambda: contract.Decision.escalate("agentseam experiment: escalate trial"), }[TRIAL]() text, code = adapter.respond(decision, event) sys.stdout.write(text) diff --git a/tools/reference_agent.py b/tools/reference_agent.py index fd1e8cf..e5f488d 100644 --- a/tools/reference_agent.py +++ b/tools/reference_agent.py @@ -96,8 +96,13 @@ def _interpret(proc, grammar): specific = body.get("hookSpecificOutput") or {} updated = specific.get("updatedInput") decision = specific.get("permissionDecision") - if decision in (ALLOW, DENY, ASK): + if decision in (ALLOW, DENY): return decision, specific.get("permissionDecisionReason") or "", updated + if decision == ASK: + # "ask" is a documented value, but what a *headless* run does next with nobody + # there to answer it is not -- guessing here would launder that silence into + # evidence, exactly what this module exists to refuse. See tools/experiment_escalate.py. + raise Undocumented("permissionDecision=%r: the protocol does not say what a headless run does next" % (ASK,)) if decision is not None: raise Undocumented("unrecognised permissionDecision: %r" % (decision,)) if body.get("decision") == "block": From 87ba7f482a0f24c3b59893c6cd72e77aa2e5297d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:13:15 +0000 Subject: [PATCH 04/29] matrix_evidence: the grade cap honours verified.observed, not just row basis claim_basis() fell back to a row's own live-run-partial basis for any event the row claims, whether or not that event was ever watched -- and W53's mechanical seeding gave every cell, watched or not, a per-claim record copying the row basis verbatim, so the fallback path was not even needed to trigger it. A live-run-partial row could therefore back `enforced` at an event its own `verified.observed` never names, which is exactly the "partial" word's whole point to prevent. claim_basis() now checks `observed` itself: when the resolved basis is live-run-partial and the event is not in it, the effective basis is the row's new optional `verified.fallback_basis` (what the rest of the row rests on), defaulting to vendor-docs when the row does not say one. Set from each row's own method text: codex_cli and vscode_copilot say source (vendor-source); cursor says vendor hooks documentation (vendor-docs). Recomputed every (agent, event) pair's basis under the fix: 11 change (the unobserved events on these three rows, all now vendor-source or vendor-docs instead of live-run-partial), all detailed in the PR body. None of the 91 claimed pairs' actual *grade* moves -- none of those 11 cells asserts a fail-closed claim, so every one was already sitting at or below its new, lower ceiling. This closes the gap defensively, same shape as W53's original cap. Co-Authored-By: Claude Sonnet 5 Signed-off-by: Claude --- src/agentseam/data/matrix.json | 9 ++++++--- src/agentseam/matrix_evidence.py | 26 +++++++++++++++++++++++--- tests/test_matrix_evidence.py | 26 +++++++++++++++++++++++++- 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/src/agentseam/data/matrix.json b/src/agentseam/data/matrix.json index 9dcd2b6..88979b8 100644 --- a/src/agentseam/data/matrix.json +++ b/src/agentseam/data/matrix.json @@ -397,7 +397,8 @@ "prompt_submit", "session_start", "stop" - ] + ], + "fallback_basis": "vendor-docs" }, "events": { "pre_tool": { @@ -1285,7 +1286,8 @@ "prompt_submit", "session_start", "stop" - ] + ], + "fallback_basis": "vendor-source" }, "events": { "pre_tool": { @@ -2023,7 +2025,8 @@ "prompt_submit", "session_start", "stop" - ] + ], + "fallback_basis": "vendor-source" }, "events": { "pre_tool": { diff --git a/src/agentseam/matrix_evidence.py b/src/agentseam/matrix_evidence.py index b11212e..f3e64b2 100644 --- a/src/agentseam/matrix_evidence.py +++ b/src/agentseam/matrix_evidence.py @@ -9,7 +9,14 @@ from __future__ import annotations from ._data import load -from .matrix_terms import BASES, CLAIM_FIELDS, OPTIONAL_CLAIM_FIELDS, REQUIRED_CLAIM_FIELDS +from .matrix_terms import ( + BASES, + BASIS_DOCS, + BASIS_LIVE_PARTIAL, + CLAIM_FIELDS, + OPTIONAL_CLAIM_FIELDS, + REQUIRED_CLAIM_FIELDS, +) _RAW = load("matrix.json") @@ -39,8 +46,21 @@ def claim_record(row, event, field): def claim_basis(row, event, field): - """The basis backing one (event, field) claim -- what `enforcement_level` caps a grade by.""" - return claim_record(row, event, field).get("basis") + """The basis backing one (event, field) claim -- what `enforcement_level` caps a grade by. + + A row basis of `live-run-partial` only backs the events its own `verified.observed` + actually names -- that is the whole point of the "partial" word. An event the row + claims but never watched falls back to `verified.fallback_basis` (what the rest of the + row rests on: source, documentation, ...), defaulting to `vendor-docs` when the row + does not say. Without this, a claim seeded mechanically from the row (or one that + simply inherits the row for lack of its own record) would let an unobserved event grade + as high as `enforced`, which is exactly the class of bug `cap_grade` exists to close. + """ + basis = claim_record(row, event, field).get("basis") + verified = row.get("verified") or {} + if basis == BASIS_LIVE_PARTIAL and event not in verified.get("observed", ()): + return verified.get("fallback_basis", BASIS_DOCS) + return basis def validate_claim(record): diff --git a/tests/test_matrix_evidence.py b/tests/test_matrix_evidence.py index f3b474f..2a4e2ed 100644 --- a/tests/test_matrix_evidence.py +++ b/tests/test_matrix_evidence.py @@ -7,7 +7,7 @@ import agentseam as A # noqa: E402 from agentseam.matrix_evidence import validate_cell, validate_claim # noqa: E402 -from agentseam.matrix_terms import BASIS_DOCS, BASIS_LIVE, FAIL_CLOSED # noqa: E402 +from agentseam.matrix_terms import BASIS_DOCS, BASIS_LIVE, BASIS_LIVE_PARTIAL, FAIL_CLOSED # noqa: E402 def test_every_asserted_cell_field_carries_evidence(): @@ -67,3 +67,27 @@ def test_a_live_run_backs_whatever_grade_the_cell_computes(): assert A.enforcement_level("_live_fixture", A.PRE_TOOL) == "enforced" finally: del A.MATRIX["_live_fixture"] + + +def test_a_partial_live_run_only_backs_enforced_at_the_events_it_observed(): + """Task 3, W55: `live-run-partial` capped every unwatched event to `enforced` too, since + a per-claim record seeded from the row (or one that simply falls back to it) carries the + row's own basis regardless of `observed`. A fail-closed `stop` cell this row never + watched must grade no higher than `best-effort` (the default fallback, `vendor-docs`, + since this fixture sets no `fallback_basis` of its own).""" + record = {"basis": BASIS_LIVE_PARTIAL, "date": "2026-09-01", "method": "fixture"} + cell = { + "block": True, + "rewrite": False, + "fail_mode": FAIL_CLOSED, + "evidence": dict.fromkeys(("block", "rewrite", "fail_mode"), record), + } + A.MATRIX["_observed_fixture"] = { + "verified": dict(record, observed=["pre_tool"]), + "events": {"pre_tool": cell, "stop": cell}, + } + try: + assert A.enforcement_level("_observed_fixture", A.PRE_TOOL) == "enforced" + assert A.enforcement_level("_observed_fixture", A.STOP) == "best-effort" + finally: + del A.MATRIX["_observed_fixture"] From 2ea1a1045998efd81d72a58bfa7da1f00e1cb8ea Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:14:38 +0000 Subject: [PATCH 05/29] changelog: report event, escalate trial, grade cap honours observed Co-Authored-By: Claude Sonnet 5 Signed-off-by: Claude --- CHANGELOG.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0268fb4..3ae06a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,39 @@ versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html). fresher per-claim record. `tools/watch_versions.py` now reads a per-claim version where one exists, so `pre_tool` reads fresh while `prompt_submit` and `stop` -- not re-run -- still show their original drift. +- **A submitted evidence report records which canonical event it measured** + (`src/agentseam/evidence_report.py`, `tools/experiment_report.py`). `as_report()` reads + `event` off the trial results instead of leaving it implicit; `evidence_report.py` + accepts it as an optional field, validated against `contract.EVENTS` when present, and + carries it through `to_evidence()`/`diff_against()` so a `pre_tool` report cannot be + merged into a `stop` claim by accident. Existing reports with no `event` stay valid. The + `evidence-report.yml` issue template gains a matching optional field. +- **An `escalate` trial** (`tools/experiment_probe.py`, `tools/experiment_escalate.py`, + `tools/experiment_driver.py`). The probe answers with `Decision.escalate()` rendered + through `adapter.respond()`, in the agent's own dialect -- Cursor's reply spells it + `ask`; an agent whose gate does not honour escalate gets its own degraded-block dialect. + The new measured field `escalate_means` (`matrix_terms.OPTIONAL_CLAIM_FIELDS`, same + "absence is not a claim" rule as the three fields above) can read `prompted` -- the run + ended waiting on an answer nobody gave -- as well as `allow` and `refusal-or-error`; + classification reads the sentinel plus the driver's own outcome (a real headless driver + that exits non-zero or times out with the sentinel untouched reads as `prompted`). The + reference driver raises `Undocumented` for `PreToolUse`'s `permissionDecision: "ask"`, + the same discipline as the unknown-verb trial: the value is documented, but what a + headless run does next with nobody there to answer is not. + +### Changed +- **The grade cap honours `verified.observed`, not just a row's basis** + (`src/agentseam/matrix_evidence.py`). A `live-run-partial` row's `claim_basis()` used to + fall back to the row's own basis for any event the row claims, watched or not, letting an + event the row never observed back a grade as high as `enforced` -- the very thing + "partial" is supposed to prevent. `claim_basis()` now falls back to the row's new + optional `verified.fallback_basis` (defaulting to `vendor-docs`) whenever the resolved + basis is `live-run-partial` and the event is absent from `observed`. Set on the three + rows that need it, from each row's own method text: `codex_cli` and `vscode_copilot` say + source (`vendor-source`); `cursor` says vendor hooks documentation (`vendor-docs`). 11 of + 91 claimed (agent, event) pairs change basis under the fix (every unobserved event on + these three rows); none changes *grade* -- none of those cells asserts a fail-closed + claim, so each was already at or below its new, lower ceiling. Full list in PR body. ### Fixed - **The stop gate's block observable is the hook re-firing, not a second action run** From ee81a7cc295d5170fb617ecaf23e387da581a184 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:15:24 +0000 Subject: [PATCH 06/29] adapters: a client_type that names another vendor beats accept_names Finding raw[8].findings[8] of the vendor-truth review, reproduced by execution. devin's accept_names claimed PermissionRequest before any marker check, on the ground that Claude Code never sends that name -- but Kimi Code does, so a real Kimi PermissionRequest was claimed by two adapters, detect() returned None, and handle() allowed it with no Event at all. Not even the observation value of the mapping survived, which contradicts the recorded rule that a positive self-identification beats a shared event name. claims.reject_client_types is checked ahead of accept_names and lists the one recorded collision. A Devin PermissionRequest, which carries no client_type, is claimed exactly as before -- pinned by a test that must keep passing. Signed-off-by: Claude Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 11 +++++++++++ src/agentseam/adapters/_payload.py | 20 +++++++++++++------- src/agentseam/data/vendors/devin.json | 5 ++++- src/agentseam/data/vendors/schema.json | 4 ++++ tests/test_adapter_devin.py | 13 +++++++++++++ tools/recount/tables.py | 7 +++++-- 6 files changed, 50 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 340e49d..aa30d4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,17 @@ versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html). re-run -- still show their original drift. ### Fixed +- **A Kimi Code `PermissionRequest` is no longer claimed by Devin as well, leaving the + payload unidentified** (`adapters/_payload.py`, `data/vendors/devin.json`; vendor-truth + review finding `raw[8].findings[8]`). Devin's `accept_names` claimed `PermissionRequest` + and `PostCompaction` before any marker check, on the ground that Claude Code never sends + those names -- but Kimi Code does send `PermissionRequest`, so a real Kimi payload was + claimed by two adapters, `detect()` returned `None`, and `handle()` allowed it with no + `Event` at all: not even the observation value survived. Reproduced by execution before + the fix. A new `claims.reject_client_types` key is checked ahead of `accept_names`, so + the CHANGELOG's own recorded rule -- a positive self-identification beats a shared event + name -- now holds for the one recorded collision. Devin's own `PermissionRequest`, which + carries no `client_type`, is claimed exactly as before. - **`PostCompact` no longer masquerades as canonical `pre_compact` on Grok and Kimi Code** (`data/vendors/grok.json`, `data/vendors/kimi_code.json`; vendor-truth review finding `raw[14].findings[12]`). Both entries mapped the vendor's diff --git a/src/agentseam/adapters/_payload.py b/src/agentseam/adapters/_payload.py index 32b442c..d5eec88 100644 --- a/src/agentseam/adapters/_payload.py +++ b/src/agentseam/adapters/_payload.py @@ -41,21 +41,27 @@ def _accepted_by_markers(c, raw, name): ) +def _disqualified(cfg, c, raw, name): + """Every reason an entry declines a payload once its wire event name is known.""" + if name not in cfg["events"]: + return True + if "client_types" in c and raw.get("client_type") not in c["client_types"]: + return True + return _rejected_by_markers(c, raw) + + def hj_claims(cfg, raw): """True when this payload matches the entry's marker discipline.""" if not isinstance(raw, dict): return False c = cfg["claims"] name = _wire_name(cfg, raw) + # A positive self-identification beats a shared event name, accept_names included. + if raw.get("client_type") in c.get("reject_client_types", ()): + return False if name in c.get("accept_names", ()): return True - if name not in cfg["events"]: - return False - if "client_types" in c and raw.get("client_type") not in c["client_types"]: - return False - if _rejected_by_markers(c, raw): - return False - return _accepted_by_markers(c, raw, name) + return not _disqualified(cfg, c, raw, name) and _accepted_by_markers(c, raw, name) def _segment(node, part): diff --git a/src/agentseam/data/vendors/devin.json b/src/agentseam/data/vendors/devin.json index 8c03d4a..3042f4a 100644 --- a/src/agentseam/data/vendors/devin.json +++ b/src/agentseam/data/vendors/devin.json @@ -12,7 +12,10 @@ "hook_event_name" ], "mode": "marker", - "notes": "accept_names are names Claude Code never sends, claimed before any marker check; prompt_id is required alongside looks_like_claude_code(raw) being false.", + "notes": "accept_names are names Claude Code never sends, claimed before any marker check -- except against a client_type that names another vendor, since Kimi Code sends PermissionRequest too; prompt_id is required alongside looks_like_claude_code(raw) being false.", + "reject_client_types": [ + "kimi_code_cli" + ], "reject_probes": [ "looks_like_claude_code" ] diff --git a/src/agentseam/data/vendors/schema.json b/src/agentseam/data/vendors/schema.json index 6f0b839..c825a9b 100644 --- a/src/agentseam/data/vendors/schema.json +++ b/src/agentseam/data/vendors/schema.json @@ -310,6 +310,10 @@ "additionalProperties": {"$ref": "#/$defs/stringList"}, "description": "vendor event name -> keys that must ALL be present for that event to be claimed when no accept_marker is (codex_cli's SessionStart compound check)." }, + "reject_client_types": { + "$ref": "#/$defs/stringList", + "description": "client-type values that disqualify the payload outright, checked before `accept_names`: a positive self-identification beats a shared event name (devin's PermissionRequest, which kimi_code also sends)." + }, "reject_markers": {"$ref": "#/$defs/stringList"}, "reject_markers_unless_probe": { "type": "object", diff --git a/tests/test_adapter_devin.py b/tests/test_adapter_devin.py index efb52e9..018ed3b 100644 --- a/tests/test_adapter_devin.py +++ b/tests/test_adapter_devin.py @@ -91,3 +91,16 @@ def test_post_compaction_is_not_bent_into_pre_compact(): assert A.PRE_COMPACT not in mod.REVERSE_EVENT_MAP assert mod.parse({"hook_event_name": "PostCompaction", "prompt_id": "turn-1"}).event == A.UNKNOWN assert A.PRE_COMPACT not in A.MATRIX["devin"]["events"] + + +def test_a_kimi_permission_request_is_left_to_kimi(): + """PermissionRequest is claimed before any marker check because Claude Code never sends""" + payload = {"hook_event_name": "PermissionRequest", "client_type": "kimi_code_cli", "tool_name": "Bash"} + assert not A.adapters.get("devin").claims(payload) + assert A.adapters.detect(payload) == "kimi_code" + + +def test_a_devin_permission_request_is_still_claimed_unconditionally(): + """The narrowing is one recorded client_type, not a retreat from accept_names.""" + payload = {"hook_event_name": "PermissionRequest", "tool_name": "Bash"} + assert A.adapters.get("devin").claims(payload) diff --git a/tools/recount/tables.py b/tools/recount/tables.py index 127858b..f7e3855 100644 --- a/tools/recount/tables.py +++ b/tools/recount/tables.py @@ -63,10 +63,13 @@ "event_key": ["hook_event_name"], "accept_markers": ["prompt_id"], "accept_names": ["PermissionRequest", "PostCompaction"], + "reject_client_types": ["kimi_code_cli"], "reject_probes": ["looks_like_claude_code"], "notes": ( - "accept_names are names Claude Code never sends, claimed before any marker check; " - "prompt_id is required alongside looks_like_claude_code(raw) being false." + "accept_names are names Claude Code never sends, claimed before any marker check " + "-- except against a client_type that names another vendor, since Kimi Code sends " + "PermissionRequest too; prompt_id is required alongside looks_like_claude_code(raw) " + "being false." ), }, "kimi_code": { From 8e9e57504f01d9b827b10944f24d6c3a85bd7083 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:16:41 +0000 Subject: [PATCH 07/29] adapters: claim a self-identified Kimi payload whatever event it names Finding raw[8].findings[6] of the vendor-truth review, reproduced by execution. claims() required the wire event name to be in `events`, so a payload carrying client_type=kimi_code_cli and a new or unmapped event name was claimed by no adapter: handle() returned event=None rather than the UNKNOWN Event the contract documents as the whole point of that pathway. Vendor drift on Kimi was invisible to a caller logging UNKNOWN events -- the one thing that pathway exists to make visible. claims.accept_any_name is opt-in and set on the one entry whose client_types cannot be null, so the relaxation rests on a positive self-identification rather than on a guess. A payload without client_type is still not claimed, and the decision is unchanged: an UNKNOWN event allows. Signed-off-by: Claude Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 11 +++++++++++ src/agentseam/adapters/_payload.py | 2 +- src/agentseam/data/vendors/kimi_code.json | 4 +++- src/agentseam/data/vendors/schema.json | 4 ++++ tests/test_adapter_kimi_code.py | 16 ++++++++++++++++ tools/recount/tables.py | 6 ++++++ 6 files changed, 41 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa30d4e..9dd7079 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,17 @@ versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html). re-run -- still show their original drift. ### Fixed +- **A Kimi Code payload naming an event Kimi has not mapped yet now reaches the caller** + (`adapters/_payload.py`, `data/vendors/kimi_code.json`; vendor-truth review finding + `raw[8].findings[6]`). `claims()` required the event name to already be in `events`, so a + payload that had positively self-identified as Kimi (`client_type: kimi_code_cli`) while + naming a new or unmapped vendor event was claimed by no adapter at all: `handle()` + returned `event=None` ("unrecognized payload") instead of the UNKNOWN `Event` the + contract documents as the whole point of that pathway ("New vendor events appear without + warning; being told is the only safe outcome"). Vendor drift on Kimi was therefore + invisible to a caller logging UNKNOWN events. A new opt-in `claims.accept_any_name` is + set on the one entry whose `client_types` cannot be null; a payload with no `client_type` + is still not claimed, and the decision itself is unchanged (an UNKNOWN event allows). - **A Kimi Code `PermissionRequest` is no longer claimed by Devin as well, leaving the payload unidentified** (`adapters/_payload.py`, `data/vendors/devin.json`; vendor-truth review finding `raw[8].findings[8]`). Devin's `accept_names` claimed `PermissionRequest` diff --git a/src/agentseam/adapters/_payload.py b/src/agentseam/adapters/_payload.py index d5eec88..03fcfd3 100644 --- a/src/agentseam/adapters/_payload.py +++ b/src/agentseam/adapters/_payload.py @@ -43,7 +43,7 @@ def _accepted_by_markers(c, raw, name): def _disqualified(cfg, c, raw, name): """Every reason an entry declines a payload once its wire event name is known.""" - if name not in cfg["events"]: + if name not in cfg["events"] and not (c.get("accept_any_name") and isinstance(name, str)): return True if "client_types" in c and raw.get("client_type") not in c["client_types"]: return True diff --git a/src/agentseam/data/vendors/kimi_code.json b/src/agentseam/data/vendors/kimi_code.json index 9826345..0b04a64 100644 --- a/src/agentseam/data/vendors/kimi_code.json +++ b/src/agentseam/data/vendors/kimi_code.json @@ -1,13 +1,15 @@ { "agent": "kimi_code", "claims": { + "accept_any_name": true, "client_types": [ "kimi_code_cli" ], "event_key": [ "hook_event_name" ], - "mode": "marker" + "mode": "marker", + "notes": "client_type cannot be absent here, so a payload carrying it has positively self-identified and is claimed whatever event name it names; parse() resolves an unmapped name to UNKNOWN, which is how Kimi's vendor drift reaches a caller." }, "config_format": "toml", "config_path": "~/.kimi-code/config.toml", diff --git a/src/agentseam/data/vendors/schema.json b/src/agentseam/data/vendors/schema.json index c825a9b..bc91469 100644 --- a/src/agentseam/data/vendors/schema.json +++ b/src/agentseam/data/vendors/schema.json @@ -305,6 +305,10 @@ "$ref": "#/$defs/stringList", "description": "vendor event names accepted unconditionally, before any other check -- names no other vendor sends (devin's PermissionRequest/PostCompaction)." }, + "accept_any_name": { + "type": "boolean", + "description": "claim an event name absent from `events` once the payload has positively self-identified, so vendor drift reaches the caller as an UNKNOWN Event rather than as no adapter at all. Valid only where `client_types` cannot be null; pinned by tests/test_adapter_kimi_code.py." + }, "accept_when_all": { "type": "object", "additionalProperties": {"$ref": "#/$defs/stringList"}, diff --git a/tests/test_adapter_kimi_code.py b/tests/test_adapter_kimi_code.py index ecfd71f..135cb42 100644 --- a/tests/test_adapter_kimi_code.py +++ b/tests/test_adapter_kimi_code.py @@ -141,3 +141,19 @@ def test_post_compact_is_not_bent_into_pre_compact(): assert mod.parse({"hook_event_name": "PreCompact", "client_type": "kimi_code_cli"}).event == A.PRE_COMPACT assert mod.REVERSE_EVENT_MAP[A.PRE_COMPACT] == "PreCompact" assert mod.claims(post), "unmapping must not blind a caller to the payload" + + +def test_an_unmapped_event_from_a_self_identified_payload_is_still_kimi(): + """client_type is Kimi's whole detection, so a name it has not mapped yet is Kimi's too:""" + novel = {"client_type": "kimi_code_cli", "hook_event_name": "TurnStarted"} + mod = A.adapters.get("kimi_code") + assert mod.claims(novel) and mod.parse(novel).event == A.UNKNOWN + _, code, event, _ = A.handle(novel, lambda _e: Decision.deny("x")) + assert (event.agent, event.event, code) == ("kimi_code", A.UNKNOWN, 0) + + +def test_accept_any_name_rests_on_client_type_never_being_absent(): + """The opt-in is safe only because a payload without client_type is not claimed at all.""" + claims = A.adapters.get("kimi_code").CONFIG["claims"] + assert claims["accept_any_name"] and None not in claims["client_types"] + assert not A.adapters.get("kimi_code").claims({"hook_event_name": "TurnStarted"}) diff --git a/tools/recount/tables.py b/tools/recount/tables.py index f7e3855..3be5441 100644 --- a/tools/recount/tables.py +++ b/tools/recount/tables.py @@ -76,6 +76,12 @@ "mode": "marker", "event_key": ["hook_event_name"], "client_types": ["kimi_code_cli"], + "accept_any_name": True, + "notes": ( + "client_type cannot be absent here, so a payload carrying it has positively " + "self-identified and is claimed whatever event name it names; parse() resolves an " + "unmapped name to UNKNOWN, which is how Kimi's vendor drift reaches a caller." + ), }, "junie": { "mode": "marker", From 8be623c43be0ef6f5cc81858bd15e5f36bf489cd Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:19:33 +0000 Subject: [PATCH 08/29] vendors: devin's degraded-rewrite note names the event, not a tool call Finding raw[5].findings[8] of the vendor-truth review, reproduced by execution: a rewrite degraded at UserPromptSubmit or Stop was refused with "Devin cannot modify a tool call", where no tool call was ever involved -- pointing an operator debugging why prompt sanitisation blocks at tool plumbing. This string is the one surface an end user reads. The note becomes the engine's template form and names the vendor event, matching cursor's per-gate phrasing. Four frozen wire bytes move (rewrite and rewrite-without-input at prompt_submit and stop); the PreToolUse and PermissionRequest bodies are byte-identical, and the reason default loses its "before it can run" tail for the same reason. Signed-off-by: Claude Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 9 +++++++++ src/agentseam/data/vendors/devin.json | 4 ++-- tests/fixtures/golden/devin.json | 8 ++++---- tests/test_adapter_devin.py | 8 ++++++++ tools/recount/tables.py | 4 ++-- 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dd7079..59862e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,15 @@ versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html). re-run -- still show their original drift. ### Fixed +- **Devin's degraded-rewrite note no longer describes a tool call at events that have + none** (`data/vendors/devin.json`; vendor-truth review finding `raw[5].findings[8]`). A + rewrite the dispatcher degraded at `UserPromptSubmit` or `Stop` was refused with + "Devin cannot modify a tool call" -- pointing an operator debugging why prompt + sanitisation blocks instead of rewriting at tool plumbing that was never involved. The + note is now the engine's template form and names the vendor event it could not modify, + matching Cursor's per-gate phrasing. Four bytes of frozen wire output move + (`tests/fixtures/golden/devin.json`: rewrite and rewrite-without-input at prompt_submit + and stop); `PreToolUse` and `PermissionRequest` output is unchanged. - **A Kimi Code payload naming an event Kimi has not mapped yet now reaches the caller** (`adapters/_payload.py`, `data/vendors/kimi_code.json`; vendor-truth review finding `raw[8].findings[6]`). `claims()` required the event name to already be in `events`, so a diff --git a/src/agentseam/data/vendors/devin.json b/src/agentseam/data/vendors/devin.json index 3042f4a..469731a 100644 --- a/src/agentseam/data/vendors/devin.json +++ b/src/agentseam/data/vendors/devin.json @@ -127,7 +127,7 @@ "default_wire_event": "PreToolUse", "degrade_notes": { "escalate": "Devin cannot prompt for confirmation, so this is a block", - "escalate_from_transform": "Devin cannot modify a tool call, so this is a block" + "escalate_from_transform": "%s (Devin cannot modify the input at %s, so this is a block)" }, "echo": "payload", "gates": { @@ -154,7 +154,7 @@ }, "note_style": "suffix", "reason_defaults": { - "transform": "input requires modification before it can run" + "transform": "input requires modification" }, "transform_grammar": "hook_specific_updated_input", "vocabulary": [ diff --git a/tests/fixtures/golden/devin.json b/tests/fixtures/golden/devin.json index ac06743..111f925 100644 --- a/tests/fixtures/golden/devin.json +++ b/tests/fixtures/golden/devin.json @@ -93,11 +93,11 @@ }, "rewrite": { "exit": 0, - "stdout": "{\"decision\": \"block\", \"reason\": \"redacted (Devin cannot modify a tool call, so this is a block)\"}" + "stdout": "{\"decision\": \"block\", \"reason\": \"redacted (Devin cannot modify the input at UserPromptSubmit, so this is a block)\"}" }, "rewrite-without-input": { "exit": 0, - "stdout": "{\"decision\": \"block\", \"reason\": \"needs change (Devin cannot modify a tool call, so this is a block)\"}" + "stdout": "{\"decision\": \"block\", \"reason\": \"needs change (Devin cannot modify the input at UserPromptSubmit, so this is a block)\"}" }, "vouch": { "exit": 0, @@ -195,11 +195,11 @@ }, "rewrite": { "exit": 0, - "stdout": "{\"decision\": \"block\", \"reason\": \"redacted (Devin cannot modify a tool call, so this is a block)\"}" + "stdout": "{\"decision\": \"block\", \"reason\": \"redacted (Devin cannot modify the input at Stop, so this is a block)\"}" }, "rewrite-without-input": { "exit": 0, - "stdout": "{\"decision\": \"block\", \"reason\": \"needs change (Devin cannot modify a tool call, so this is a block)\"}" + "stdout": "{\"decision\": \"block\", \"reason\": \"needs change (Devin cannot modify the input at Stop, so this is a block)\"}" }, "vouch": { "exit": 0, diff --git a/tests/test_adapter_devin.py b/tests/test_adapter_devin.py index 018ed3b..6c9eb73 100644 --- a/tests/test_adapter_devin.py +++ b/tests/test_adapter_devin.py @@ -104,3 +104,11 @@ def test_a_devin_permission_request_is_still_claimed_unconditionally(): """The narrowing is one recorded client_type, not a retreat from accept_names.""" payload = {"hook_event_name": "PermissionRequest", "tool_name": "Bash"} assert A.adapters.get("devin").claims(payload) + + +def test_a_degraded_rewrite_names_the_event_it_could_not_modify(): + """At prompt_submit and stop there is no tool call, so a note about one sends the""" + payload = {"hook_event_name": "UserPromptSubmit", "prompt_id": "p1", "prompt": "remember my key"} + text, _, _, _ = A.handle(payload, lambda _e: Decision.rewrite({"prompt": "clean"}, "sanitized")) + reason = json.loads(text)["reason"] + assert "tool call" not in reason and "at UserPromptSubmit" in reason diff --git a/tools/recount/tables.py b/tools/recount/tables.py index 3be5441..d9b3bdc 100644 --- a/tools/recount/tables.py +++ b/tools/recount/tables.py @@ -164,9 +164,9 @@ "words": {"allow": "approve", "block": "block"}, "degrade_notes": { "escalate": "Devin cannot prompt for confirmation, so this is a block", - "escalate_from_transform": "Devin cannot modify a tool call, so this is a block", + "escalate_from_transform": "%s (Devin cannot modify the input at %s, so this is a block)", }, - "reason_defaults": {"transform": "input requires modification before it can run"}, + "reason_defaults": {"transform": "input requires modification"}, "note_style": "suffix", "echo": "payload", "default_wire_event": "PreToolUse", From 44647b6c10e69b47e107b0119c09e1c73f098f09 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 19:10:20 -0400 Subject: [PATCH 09/29] tests: drop the "expanduser reads HOME" assumption from the user-scoped path test test_a_user_scoped_config_path_is_not_nested_under_the_repo set HOME to tmp_path itself and then asserted the config landed in tmp_path/.junie. Only posixpath.expanduser reads HOME. ntpath reads USERPROFILE, then HOMEDRIVE + HOMEPATH, and conftest's autouse isolated_home fixture already points all four at a `home` directory beneath tmp_path -- so on Windows the install correctly wrote its config under the fixture's home while the assertion looked one level up, in tmp_path itself. The fixture is the platform-independent answer and the test now takes it as a parameter instead of re-deriving home from an env var that three quarters of the supported platforms ignore. Signed-off-by: Claude Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_install.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_install.py b/tests/test_install.py index 14da650..ffe6f70 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -118,13 +118,16 @@ def test_cursor_install_writes_the_generic_gate_with_fail_closed(tmp_path): assert I.uninstall("cursor", root) is True -def test_a_user_scoped_config_path_is_not_nested_under_the_repo(tmp_path, monkeypatch): - """`~/...` means the user's home, not a directory literally named `~` in the repo.""" - monkeypatch.setenv("HOME", str(tmp_path)) +def test_a_user_scoped_config_path_is_not_nested_under_the_repo(tmp_path, isolated_home): + """`~/...` means the user's home, not a directory literally named `~` in the repo. + + Home comes from the `isolated_home` fixture, not a local `setenv("HOME")`: only + posixpath reads HOME, so setting it by hand pointed this assertion at a directory + Windows never expands to.""" written = I.install("junie", ["pre_tool"], "guard.py", str(tmp_path)) assert not (tmp_path / "~").exists(), "created a directory literally named ~" - assert Path(written) == tmp_path / ".junie" / "config.json" + assert Path(written) == isolated_home / ".junie" / "config.json" assert I.installed("junie", str(tmp_path)) From 3dfd42201cc562ec6c4aa2fafa9ff0bc25b7b25a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 19:10:48 -0400 Subject: [PATCH 10/29] tests: drop the "expanduser reads HOME" assumption from the data-loss test test_install_never_destroys_a_config_it_cannot_parse seeded a config at tmp_path/.junie/config.json after setting HOME to tmp_path, then called install() and checked the user's settings survived. On Windows expanduser resolved ~ to the fixture's tmp_path/home, so install never saw the seeded file at all: it created a fresh config elsewhere, and every assertion here was reading a file nothing had touched. The BOM case passed vacuously and the unparseable cases raised JSONDecodeError from the test's own json.loads rather than ConfigUnreadableError from install. Home now comes from the isolated_home fixture, which sets every variable expanduser actually consults on both platform families. Signed-off-by: Claude Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_install.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_install.py b/tests/test_install.py index ffe6f70..3b327cc 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -131,10 +131,13 @@ def test_a_user_scoped_config_path_is_not_nested_under_the_repo(tmp_path, isolat assert I.installed("junie", str(tmp_path)) -def test_install_never_destroys_a_config_it_cannot_parse(tmp_path, monkeypatch): - """The data-loss bug: _load returned {} on any parse failure, so install merged its""" - monkeypatch.setenv("HOME", str(tmp_path)) - cfg = tmp_path / ".junie" / "config.json" +def test_install_never_destroys_a_config_it_cannot_parse(isolated_home): + """The data-loss bug: _load returned {} on any parse failure, so install merged its + + Home is the fixture's, not a local setenv("HOME"): ntpath.expanduser never reads HOME, + so the seeded config sat where no install would ever look and every assertion below + was measuring an untouched file rather than a preserved one.""" + cfg = isolated_home / ".junie" / "config.json" cfg.parent.mkdir() cfg.write_bytes(b"\xef\xbb\xbf" + json.dumps({"theme": "dark", "customModel": "keep-me"}).encode()) From 6019f02496ba8d73314a06ddd548797351539cc4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 19:11:13 -0400 Subject: [PATCH 11/29] tests: drop the "expanduser reads HOME" assumption from the read-only query test test_a_query_never_raises_on_an_unparseable_config wrote a corrupt config under a HOME it had set by hand, then asserted installed() returns False and uninstall() raises ConfigUnreadableError. On Windows the corrupt file was not on the path either call reads, so installed() returned False because nothing was there -- the right answer for the wrong reason -- and uninstall() found no file to fail on, which is why the test reported DID NOT RAISE. Home now comes from the isolated_home fixture. All three install tests that seeded a user-scoped config by hand are now consistent with it. Signed-off-by: Claude Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_install.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_install.py b/tests/test_install.py index 3b327cc..ff2d0cb 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -157,10 +157,13 @@ def test_install_never_destroys_a_config_it_cannot_parse(isolated_home): assert cfg.read_bytes()[:2] == b"\xff\xfe", "a UTF-16 config was overwritten" -def test_a_query_never_raises_on_an_unparseable_config(tmp_path, monkeypatch): - """installed() is a read-only question; a corrupt file means "not known to be there",""" - monkeypatch.setenv("HOME", str(tmp_path)) - cfg = tmp_path / ".junie" / "config.json" +def test_a_query_never_raises_on_an_unparseable_config(isolated_home): + """installed() is a read-only question; a corrupt file means "not known to be there", + + Home is the fixture's for the reason the two tests above give: a local setenv("HOME") + is read by posixpath only, so the corrupt file was never on the path uninstall reads + and the "must raise" assertion had nothing to raise about.""" + cfg = isolated_home / ".junie" / "config.json" cfg.parent.mkdir() cfg.write_text("{ broken ,,, }") From 81015483b5f4435bdc45b97b8bfafb9f9824d426 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 19:11:36 -0400 Subject: [PATCH 12/29] tests: drop the "expanduser reads HOME" assumption from the kimi install test kimi_code's CONFIG_PATH starts with ~/, so its config is user-scoped by design -- there is a test in this same file asserting exactly that. The install test still seeded the file at tmp_path/.kimi-code/config.toml after setting HOME by hand, which resolves nowhere on Windows. The result was the sharpest form of the failure: install() wrote a correct config under the fixture's home, the test read the untouched original it had seeded, and reported that guard.py was missing from a file install had never been asked to write. Home now comes from the isolated_home fixture. tmp_path stays as the repo root argument, which is a different thing and still belongs there. Signed-off-by: Claude Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_adapter_kimi_code.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_adapter_kimi_code.py b/tests/test_adapter_kimi_code.py index bf7eb85..335c517 100644 --- a/tests/test_adapter_kimi_code.py +++ b/tests/test_adapter_kimi_code.py @@ -94,10 +94,13 @@ def test_a_command_with_quotes_survives_the_toml_round_trip(): assert 'command = "sh -c \\"echo hi\\""' in toml -def test_install_appends_a_block_and_leaves_the_users_settings_untouched(tmp_path, monkeypatch): - """config.toml is the user's whole CLI configuration, not a hooks file.""" - monkeypatch.setenv("HOME", str(tmp_path)) - config = Path(tmp_path) / ".kimi-code" / "config.toml" +def test_install_appends_a_block_and_leaves_the_users_settings_untouched(tmp_path, isolated_home): + """config.toml is the user's whole CLI configuration, not a hooks file. + + Home comes from the `isolated_home` fixture: kimi_code's config is user-scoped, and a + local setenv("HOME") is read by posixpath only, so the seeded file sat where no install + would look and the "left untouched" assertion was reading an untouched file.""" + config = isolated_home / ".kimi-code" / "config.toml" config.parent.mkdir(parents=True) original = '[model]\nname = "kimi-k2"\n\n[[hooks]]\nevent = "Stop"\ncommand = "mine.sh"\n' config.write_text(original) From 09ce16b75ee185fff4d417dbe1b97963505f70dd Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 19:11:58 -0400 Subject: [PATCH 13/29] tests: drop the "expanduser reads HOME" assumption from the kimi reinstall test test_reinstalling_replaces_our_block_rather_than_stacking_them read the config back from tmp_path/.kimi-code/config.toml after two installs. Both installs wrote to the home expanduser actually resolves, so on Windows the read raised FileNotFoundError before any assertion about stacked blocks ran. Home now comes from the isolated_home fixture, so the file the test reads is the file the installs wrote. `Path` is still used elsewhere in the module, so the import stays. Signed-off-by: Claude Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_adapter_kimi_code.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_adapter_kimi_code.py b/tests/test_adapter_kimi_code.py index 335c517..dc6dddb 100644 --- a/tests/test_adapter_kimi_code.py +++ b/tests/test_adapter_kimi_code.py @@ -115,11 +115,12 @@ def test_install_appends_a_block_and_leaves_the_users_settings_untouched(tmp_pat assert I.installed("kimi_code", str(tmp_path)) is False -def test_reinstalling_replaces_our_block_rather_than_stacking_them(tmp_path, monkeypatch): - monkeypatch.setenv("HOME", str(tmp_path)) +def test_reinstalling_replaces_our_block_rather_than_stacking_them(tmp_path, isolated_home): + """Home is the fixture's, not a local setenv("HOME"): the file both installs actually + wrote lives under the home expanduser resolves, which on Windows is never HOME.""" I.install("kimi_code", ["pre_tool"], "first.py", str(tmp_path)) I.install("kimi_code", ["pre_tool"], "second.py", str(tmp_path)) - text = (Path(tmp_path) / ".kimi-code" / "config.toml").read_text() + text = (isolated_home / ".kimi-code" / "config.toml").read_text() assert text.count(I.BEGIN) == 1 assert "second.py" in text and "first.py" not in text From 2439ce865d338b80781d1ebcd57cadee9bba563f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 19:13:20 -0400 Subject: [PATCH 14/29] examples: drop the "the platform default encoding is utf-8" assumption The generated pages carry em dashes. Both readers that compare a committed page against a freshly built one opened it with no encoding argument, so on a cp1252 host every em dash decoded to three characters that could never match. test_committed_pages_match_what_the_library_produces reported all twelve pages stale on Windows -- with no way to make it pass, because regenerating wrote the same bytes it had just failed to read back. Both readers now say encoding="utf-8": the test's, extracted into a _committed() helper that can carry the reason, and generate.py's --check path, which the pre-commit hook and the examples CI job both run and which had the identical latent bug. The writer says encoding="utf-8", newline="\n" for the same reason from the other side: a regeneration on Windows was writing CRLF into files the repo stores with LF, so every page showed as modified until git normalised it away. Same assumption, same fix, and it keeps a Windows regeneration byte-identical to a CI one. Signed-off-by: Claude Co-Authored-By: Claude Opus 5 (1M context) --- examples/generate.py | 4 ++-- tests/test_examples.py | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/examples/generate.py b/examples/generate.py index 67ecb7e..b91006e 100755 --- a/examples/generate.py +++ b/examples/generate.py @@ -266,7 +266,7 @@ def main(argv=None): drift = [] for name, body in sorted(files.items()): path = os.path.join(OUT, name) - current = open(path).read() if os.path.exists(path) else None + current = open(path, encoding="utf-8").read() if os.path.exists(path) else None if current != body: drift.append(name) sys.stderr.writelines( @@ -288,7 +288,7 @@ def main(argv=None): return 0 os.makedirs(OUT, exist_ok=True) for name, body in sorted(files.items()): - with open(os.path.join(OUT, name), "w") as fh: + with open(os.path.join(OUT, name), "w", encoding="utf-8", newline="\n") as fh: fh.write(body) print("wrote %d files to %s" % (len(files), OUT)) return 0 diff --git a/tests/test_examples.py b/tests/test_examples.py index 36c0ad4..864ed2c 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -25,12 +25,18 @@ def generated(): return build() +def _committed(name): + """utf-8, not the platform default: these pages carry em dashes, and reading them under + cp1252 mangles every one -- which reported all twelve pages as stale on Windows.""" + return open(os.path.join(OUT, name), encoding="utf-8").read() + + def test_committed_pages_match_what_the_library_produces(generated): """An example nobody regenerates is a claim nobody checks.""" stale = [ name for name, body in sorted(generated.items()) - if (open(os.path.join(OUT, name)).read() if os.path.exists(os.path.join(OUT, name)) else None) != body + if (_committed(name) if os.path.exists(os.path.join(OUT, name)) else None) != body ] assert not stale, "stale examples, run `python3 examples/generate.py`: %s" % ", ".join(stale) From c015ed99b5927d0608fe60fb57ad3b7afe4b4383 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 19:14:24 -0400 Subject: [PATCH 15/29] tests: drop the "the filesystem carries a POSIX execute bit" assumption test_hook_is_executable read HOOK.stat().st_mode & 0o111. Windows has no POSIX execute bit, so that is 0 for every file in the tree -- including this one, which git records as 100755 and checks out executable on Linux and macOS. The test could not pass on Windows and told the truth about nothing there. It now reads the mode git records for the path. That is the mode a clone gets, which is what decides whether the hook runs, and it is the same answer on every platform. A hook that is not tracked at all now fails with its own message rather than an index error. Verified real rather than vacuous: git currently records 100755 for .githooks/pre-commit, and the assertion names the mode it found when it is anything else. Signed-off-by: Claude Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_git_hooks.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/test_git_hooks.py b/tests/test_git_hooks.py index a5f04e7..d54d8b0 100644 --- a/tests/test_git_hooks.py +++ b/tests/test_git_hooks.py @@ -40,9 +40,18 @@ def _commit(clone, message): def test_hook_is_executable(): - """A hook without the bit set is skipped by git without a word.""" + """A hook without the bit set is skipped by git without a word. + + The mode is read from git's index, not from the filesystem. Windows has no POSIX + execute bit, so HOOK.stat() reports a plain 0o666 there for the very same file git + records -- and checks out on Linux and macOS -- as 100755. The recorded mode is what + travels with a clone, so it is the mode worth asserting on every platform.""" + if not shutil.which("git"): # pragma: no cover - git is present everywhere we run + pytest.skip("git unavailable") assert HOOK.exists() - assert HOOK.stat().st_mode & 0o111, "pre-commit hook is not executable" + recorded = _run(["git", "ls-files", "-s", "--", ".githooks/pre-commit"], ROOT).stdout.split() + assert recorded, "pre-commit hook is not tracked by git, so no clone will ever run it" + assert recorded[0] == "100755", "git records mode %s, not the executable 100755" % recorded[0] def test_changing_an_adapter_refreshes_the_pages_in_the_same_commit(clone): From 7c5b0c41f481bf864e9ad09d4d1625b3e17bd5ac Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 19:15:02 -0400 Subject: [PATCH 16/29] tests: remove the same HOME assumption from a test it was silently passing test_a_query_never_raises_on_an_undecodable_toml_config carried the same hand-set HOME as the three install tests fixed earlier on this branch, but it did not fail on Windows. That is the worse outcome: installed() returned False because the utf-16 file was not on the path it reads, not because the TOML branch had decoded it safely. The test asserted its own setup. Not one of the seven reported failures. It is the same defect in the same file, found while fixing them, and it was reporting a pass for a branch it never entered. With home taken from the fixture, the file is on the path installed() reads: verified by writing a valid installed config to that path and watching installed() return True, then overwriting it with the utf-16 body and watching it return False without raising. Signed-off-by: Claude Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_install.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_install.py b/tests/test_install.py index ff2d0cb..4f80475 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -173,10 +173,13 @@ def test_a_query_never_raises_on_an_unparseable_config(isolated_home): assert cfg.read_text() == "{ broken ,,, }", "uninstall must not rewrite a file it cannot parse" -def test_a_query_never_raises_on_an_undecodable_toml_config(tmp_path, monkeypatch): - """The TOML branch of installed() must uphold the same "never raises" contract as the""" - monkeypatch.setenv("HOME", str(tmp_path)) - cfg = tmp_path / ".kimi-code" / "config.toml" +def test_a_query_never_raises_on_an_undecodable_toml_config(isolated_home): + """The TOML branch of installed() must uphold the same "never raises" contract as the + + Home is the fixture's for the same reason as the three tests above. This one did not + fail on Windows, which is worse: installed() returned False because the undecodable + file was not on the path it reads, so the TOML branch under test never ran.""" + cfg = isolated_home / ".kimi-code" / "config.toml" cfg.parent.mkdir() cfg.write_bytes('event = "x"'.encode("utf-16")) From 96ec7ae23c47ee9379a288bfb378b74fe9552fe5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:24:11 +0000 Subject: [PATCH 17/29] recordings: freeze a witnessed run as data, schema-validated data/recordings/@.json holds the sentinel and hook-invocation counts a real agent produced per trial, one immutable file per (agent, version). agentseam.recordings is the package-side reader; the dev-only writer lands with the recorded driver in the next commit. Seeds claude_code@2.1.263 from the witnessed pre_tool run already on org-plan (plan/spine-a/evidence/). Co-Authored-By: Claude Sonnet 5 Signed-off-by: Claude --- pyproject.toml | 2 +- .../data/recordings/claude_code@2.1.263.json | 30 ++++++ src/agentseam/data/recordings/schema.json | 79 +++++++++++++++ src/agentseam/recordings.py | 57 +++++++++++ tests/test_recordings.py | 96 +++++++++++++++++++ 5 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 src/agentseam/data/recordings/claude_code@2.1.263.json create mode 100644 src/agentseam/data/recordings/schema.json create mode 100644 src/agentseam/recordings.py create mode 100644 tests/test_recordings.py diff --git a/pyproject.toml b/pyproject.toml index 87ca5e6..da45e43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ where = ["src"] # The vendor tables live in data/*.json and are read at import time. Without this the # wheel ships without them and every import raises FileNotFoundError from a pip install # while working perfectly from a source checkout -- the failure chock#87 caught on itself. -agentseam = ["data/*.json", "data/vendors/*.json", "data/templates/*.py.tmpl"] +agentseam = ["data/*.json", "data/vendors/*.json", "data/recordings/*.json", "data/templates/*.py.tmpl"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/agentseam/data/recordings/claude_code@2.1.263.json b/src/agentseam/data/recordings/claude_code@2.1.263.json new file mode 100644 index 0000000..a4766c1 --- /dev/null +++ b/src/agentseam/data/recordings/claude_code@2.1.263.json @@ -0,0 +1,30 @@ +{ + "agent": "claude_code", + "version": "2.1.263", + "platform": "linux", + "date": "2026-09-07", + "driver": "real-agent", + "notes": "Sentinel/invocation counts are the minimal values that reproduce the measured outcomes witnessed against real Claude Code CLI 2.1.263 (org-plan plan/spine-a/evidence/agentseam-claude_code-pre_tool-2.1.263.json), not a raw byte-for-byte log.", + "events": { + "pre_tool": { + "payload_keys": [ + "cwd", + "hook_event_name", + "permission_mode", + "session_id", + "tool_input", + "tool_name", + "transcript_path" + ], + "trials": { + "allow": { "observed": { "runs": 1, "alt_runs": 0 }, "hook_invocations": 1 }, + "deny": { "observed": { "runs": 0, "alt_runs": 0 }, "hook_invocations": 1 }, + "crash": { "observed": { "runs": 1, "alt_runs": 0 }, "hook_invocations": 1 }, + "silence": { "observed": { "runs": 1, "alt_runs": 0 }, "hook_invocations": 1 }, + "timeout": { "observed": { "runs": 1, "alt_runs": 0 }, "hook_invocations": 1 }, + "transform": { "observed": { "runs": 0, "alt_runs": 1 }, "hook_invocations": 1 }, + "unknown": { "observed": { "runs": 1, "alt_runs": 0 }, "hook_invocations": 1 } + } + } + } +} diff --git a/src/agentseam/data/recordings/schema.json b/src/agentseam/data/recordings/schema.json new file mode 100644 index 0000000..30b7770 --- /dev/null +++ b/src/agentseam/data/recordings/schema.json @@ -0,0 +1,79 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/open-coder-ai/agentseam/blob/main/src/agentseam/data/recordings/schema.json", + "title": "agentseam recorded driver: a frozen witness run", + "description": [ + "One file per (agent, version) -- data/recordings/@.json -- written by", + "tools/experiment.py --record from a real agent, never by hand and never edited after", + "landing: a newer version is a new file, not a rewrite of this one (owner decision", + "2026-09-07, org-plan plan/agentseam-project.md 'Evidence layer'). `tools/recorded_driver.py`", + "replays it through the same tools/experiment.py classifier a live run would have used,", + "so `--driver recorded` reproduces the witnessed table without launching a process.", + "Facts only: `notes` is the one line of prose this file allows." + ], + "type": "object", + "additionalProperties": false, + "required": ["agent", "version", "platform", "date", "driver", "events"], + "properties": { + "agent": { "type": "string", "pattern": "^[a-z][a-z0-9_]*$" }, + "version": { "type": "string", "minLength": 1 }, + "platform": { "type": "string", "minLength": 1 }, + "date": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}$" }, + "driver": { + "enum": ["real-agent"], + "description": "Always real-agent: the same invariant evidence_report.py enforces on a submitted report -- only a running agent may freeze a witness recording." + }, + "reporter": { "type": "string", "minLength": 1 }, + "notes": { + "type": "string", + "minLength": 1, + "description": "One line. The why belongs in the PR body, not here." + }, + "events": { + "type": "object", + "minProperties": 1, + "description": "Canonical event name -> what was witnessed there. A key present here IS the claim 'this gate was measured'; a gate this agent claims but this file omits was simply not run yet -- that is the partial-recording case, not an error.", + "additionalProperties": { "$ref": "#/$defs/eventRecord" } + } + }, + "$defs": { + "eventRecord": { + "type": "object", + "additionalProperties": false, + "required": ["trials", "payload_keys"], + "properties": { + "payload_keys": { + "type": "array", + "items": { "type": "string", "minLength": 1 }, + "description": "Every payload key seen across this event's trials, sorted and deduplicated -- the wire shape at this gate, not a per-trial value." + }, + "trials": { + "type": "object", + "minProperties": 1, + "description": "Trial name (tools/experiment_probe.py's BEHAVIOURS, plus escalate when run) -> the sentinel and invocation counts witnessed. Not the classified outcome itself: replaying these through experiment._classify is what keeps a recording's semantics identical to a live run's, rather than merely similar.", + "additionalProperties": { "$ref": "#/$defs/trialRecord" } + } + } + }, + "trialRecord": { + "type": "object", + "additionalProperties": false, + "required": ["observed", "hook_invocations"], + "properties": { + "observed": { + "type": "object", + "additionalProperties": false, + "required": ["runs", "alt_runs"], + "properties": { + "runs": { "type": "number" }, + "alt_runs": { "type": "number" } + } + }, + "hook_invocations": { + "type": "number", + "description": "How many times the hook fired for this trial. At the stop gate this is the re-fire count, capped by the harness's own max_continuations -- not a raw wall-clock count." + } + } + } + } +} diff --git a/src/agentseam/recordings.py b/src/agentseam/recordings.py new file mode 100644 index 0000000..cc8c322 --- /dev/null +++ b/src/agentseam/recordings.py @@ -0,0 +1,57 @@ +"""Frozen witness recordings: data the package reads, never writes. + +A recording under ``data/recordings/@.json`` is what +``tools/experiment.py --record`` freezes from a real agent run (``tools/recorded_driver.py``, +a dev-only tool). Reading it back -- so ``--driver recorded``, ``tools/watch_versions.py`` and +``agentseam matrix --evidence`` all agree on what is on disk -- belongs in the installed, +stdlib-only package instead, the same split every other data table in this repo follows. +""" + +from __future__ import annotations + +import os +import re + +from ._data import load +from .staleness import parse_version + +_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") +RECORDINGS_DIR = os.path.join(_DATA_DIR, "recordings") +_FILENAME_RE = re.compile(r"^([a-z][a-z0-9_]*)@(.+)\.json$") + + +def _recording_files(): + if not os.path.isdir(RECORDINGS_DIR): + return [] + return sorted(name for name in os.listdir(RECORDINGS_DIR) if _FILENAME_RE.match(name)) + + +def versions(agent): + """Every version recorded for `agent`, oldest first.""" + found = (_FILENAME_RE.match(name) for name in _recording_files()) + return sorted((m.group(2) for m in found if m and m.group(1) == agent), key=parse_version) + + +def latest_version(agent): + """The newest recorded version for `agent`, or None if it has never been witnessed.""" + found = versions(agent) + return found[-1] if found else None + + +def path_for(agent, version): + """Where `agent`'s recording at `version` lives, whether or not it exists yet.""" + return os.path.join(RECORDINGS_DIR, "%s@%s.json" % (agent, version)) + + +def load_recording(agent, version=None): + """The recording body for `agent` at `version` (default: its latest), or None.""" + version = version or latest_version(agent) + if version is None: + return None + relative = "recordings/%s@%s.json" % (agent, version) + if not os.path.exists(os.path.join(_DATA_DIR, relative)): + return None + return load(relative) + + +__all__ = ["RECORDINGS_DIR", "latest_version", "load_recording", "path_for", "versions"] diff --git a/tests/test_recordings.py b/tests/test_recordings.py new file mode 100644 index 0000000..3afc9d6 --- /dev/null +++ b/tests/test_recordings.py @@ -0,0 +1,96 @@ +"""Recording format: schema-validated, and immutable by filename convention. + +A recording is what tools/experiment.py --record freezes from a real agent run. Nothing +here exercises the harness itself (tests/test_recorded_driver.py does that) -- this is +purely about the data shape, the same split test_data_tables.py draws for every other table. +""" + +from __future__ import annotations + +import copy +import json +import os +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "tools")) + +from validate_vendor_config import validate # noqa: E402 + +from agentseam import recordings # noqa: E402 +from agentseam._data import load # noqa: E402 + +RECORDINGS_DIR = ROOT / "src" / "agentseam" / "data" / "recordings" +SCHEMA = load("recordings/schema.json") + + +def _committed_recordings(): + return sorted(p.name for p in RECORDINGS_DIR.glob("*.json") if p.name != "schema.json") + + +@pytest.mark.parametrize("name", _committed_recordings()) +def test_schema_validates_every_committed_recording(name): + assert validate(SCHEMA, load("recordings/%s" % name)) == [] + + +@pytest.mark.parametrize("name", _committed_recordings()) +def test_filename_matches_the_recording_it_names(name): + """The immutability rule (a newer version is a new file) rests on this holding.""" + body = load("recordings/%s" % name) + assert name == "%s@%s.json" % (body["agent"], body["version"]) + + +def _seed(): + with open(RECORDINGS_DIR / "claude_code@2.1.263.json", encoding="utf-8") as fh: + return json.load(fh) + + +def test_a_missing_required_header_field_fails_by_name(): + mutated = copy.deepcopy(_seed()) + del mutated["driver"] + errors = validate(SCHEMA, mutated) + assert any("driver" in e for e in errors) + + +def test_a_trial_missing_hook_invocations_fails_by_name(): + mutated = copy.deepcopy(_seed()) + del mutated["events"]["pre_tool"]["trials"]["deny"]["hook_invocations"] + errors = validate(SCHEMA, mutated) + assert any("hook_invocations" in e for e in errors) + + +def test_an_unrecognised_driver_value_is_rejected(): + mutated = copy.deepcopy(_seed()) + mutated["driver"] = "reference" + errors = validate(SCHEMA, mutated) + assert any("driver" in e for e in errors) + + +def test_an_extra_top_level_field_is_rejected(): + mutated = copy.deepcopy(_seed()) + mutated["basis"] = "live-run" + errors = validate(SCHEMA, mutated) + assert any("basis" in e for e in errors) + + +def test_benign_edits_still_pass(): + """Reordering keys and adding the optional reporter/notes fields change nothing constrained.""" + mutated = copy.deepcopy(_seed()) + mutated["reporter"] = "@someone" + mutated = dict(reversed(list(mutated.items()))) + assert validate(SCHEMA, mutated) == [] + + +def test_recordings_reader_finds_the_seeded_claude_code_version(): + assert "2.1.263" in recordings.versions("claude_code") + assert recordings.latest_version("claude_code") == "2.1.263" + assert os.path.exists(recordings.path_for("claude_code", "2.1.263")) + + +def test_recordings_reader_returns_none_for_an_agent_never_witnessed(): + assert recordings.latest_version("no-such-agent") is None + assert recordings.load_recording("no-such-agent") is None From 9c83d4310db5105519edcee41e356f057b7f94e2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:24:21 +0000 Subject: [PATCH 18/29] experiment: --record freezes a run, --driver recorded replays it tools/recorded_driver.py is the dev-only writer and replay driver: --record freezes a real-agent run into data/recordings/, refusing the reference and recorded drivers themselves (the same live-evidence invariant evidence_report.py already enforces on a submitted report). --driver recorded (default once a recording exists for --agent) replays a trial through experiment._classify with no process launched, so the recorded table reproduces in under a second instead of a subprocess round trip. evidence_report.py gains recorded_version and rejects a recorded-driver report whose version exceeds the recording it actually replayed. Co-Authored-By: Claude Sonnet 5 Signed-off-by: Claude --- src/agentseam/evidence_report.py | 25 ++++ tests/test_recorded_driver.py | 176 +++++++++++++++++++++++++++++ tools/experiment.py | 30 +++-- tools/experiment_report.py | 40 +++++-- tools/recorded_driver.py | 188 +++++++++++++++++++++++++++++++ 5 files changed, 444 insertions(+), 15 deletions(-) create mode 100644 tests/test_recorded_driver.py create mode 100644 tools/recorded_driver.py diff --git a/src/agentseam/evidence_report.py b/src/agentseam/evidence_report.py index 803f191..fad28bd 100644 --- a/src/agentseam/evidence_report.py +++ b/src/agentseam/evidence_report.py @@ -24,6 +24,7 @@ from __future__ import annotations from .matrix_terms import BASES, BASIS_DOCS, BASIS_LIVE, BASIS_LIVE_PARTIAL +from .staleness import parse_version #: Current report format. Bumped when a required field changes, so an old submission is #: rejected with "regenerate this" rather than silently half-read. @@ -44,11 +45,17 @@ "run_url", "notes", "tool_version", + "recorded_version", ) #: The driver name the harness uses for its credential-free documentation simulator. REFERENCE_DRIVER = "reference" +#: The driver name tools/recorded_driver.py replays through -- a frozen witness run, not a +#: live one. Named here, not just in that tool, so this module's own honesty rule (below) +#: does not have to import a dev-only tool to enforce it. +RECORDED_DRIVER = "recorded" + #: Bases that assert somebody watched a real agent do something. LIVE_BASES = (BASIS_LIVE, BASIS_LIVE_PARTIAL) @@ -109,12 +116,30 @@ def validate(report): "watched cannot be checked for drift later." % report["basis"] ) + if report["driver"] == RECORDED_DRIVER: + _check_recorded_version(report) + if not str(report["date"]).count("-") == 2: # noqa: PLR2004 raise InvalidReportError("date must be YYYY-MM-DD, got %r" % (report["date"],)) return report +def _check_recorded_version(report): + """A replayed recording may only claim the version it actually replayed, or older.""" + recorded_version = report.get("recorded_version") + if not recorded_version: + raise InvalidReportError( + "driver %r requires `recorded_version`: which frozen run this replayed." % RECORDED_DRIVER + ) + claimed = report.get("version") or recorded_version + if parse_version(claimed) > parse_version(recorded_version): + raise InvalidReportError( + "version %r exceeds recorded_version %r: a replayed recording cannot claim " + "evidence for a build it never ran against." % (claimed, recorded_version) + ) + + def to_evidence(report): """The matrix.json row-evidence entry a validated report becomes. diff --git a/tests/test_recorded_driver.py b/tests/test_recorded_driver.py new file mode 100644 index 0000000..e0ad240 --- /dev/null +++ b/tests/test_recorded_driver.py @@ -0,0 +1,176 @@ +"""The recorded driver: replays a frozen run instead of launching a real agent. + +Two things matter here and are asserted directly, mirroring test_experiment_kit.py's own +split: that a replay goes through the real classifier (so it cannot silently drift from +what a live run would produce), and that the record/refuse invariants around `--record` +hold -- the same "only a real agent may claim live evidence" rule evidence_report.py +enforces on a submitted report. +""" + +from __future__ import annotations + +import json +import os +import shutil +import sys +import time + +import pytest + +TOOLS = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "tools") +sys.path.insert(0, TOOLS) + +import experiment # noqa: E402 +import experiment_report # noqa: E402 +import recorded_driver # noqa: E402 + +from agentseam import evidence_report # noqa: E402 + +AGENT = "claude_code" +VERSION = "2.1.263" +WITNESSED = { + "allow": {"baseline_ok": True}, + "deny": {"block": True}, + "crash": {"fail_mode": "open"}, + "silence": {"silence_means": "allow"}, + "timeout": {"timeout_fail_mode": "open"}, + "transform": {"transform": True}, + "unknown": {"unknown_verb_means": "allow"}, +} + + +def test_the_seven_trials_reproduce_the_witnessed_table_in_under_a_second(): + started = time.time() + results = [ + experiment.run_trial(AGENT, trial, event="pre_tool", driver="recorded", agent_version=VERSION) + for trial in WITNESSED + ] + assert time.time() - started < 1.0 + for r in results: + assert r["measured"] == WITNESSED[r["trial"]], r["trial"] + assert r["driver"] == "recorded" + assert r["workspace"] is None + + +def test_replay_goes_through_the_real_classifier_not_a_shortcut(): + """A recording that lied about its counts would produce a wrong classification -- + proving the replay path is `_classify` itself, not a stored answer.""" + r = experiment.run_trial(AGENT, "deny", event="pre_tool", driver="recorded", agent_version=VERSION) + assert r["observed"] == {"runs": 0, "alt_runs": 0} + assert r["hook_invocations"] == 1 + assert r["measured"] == {"block": True} + + +def test_diff_against_matrix_agrees_on_every_trial(): + results = [ + experiment.run_trial(AGENT, trial, event="pre_tool", driver="recorded", agent_version=VERSION) + for trial in sorted(WITNESSED) + ] + rows = experiment_report.diff_against_matrix(results) + disagreements = [row for row in rows if row["status"] == "DISAGREES"] + assert disagreements == [] + + +def test_a_trial_the_recording_never_covers_raises(): + with pytest.raises(recorded_driver.NoRecording): + experiment.run_trial(AGENT, "deny", event="stop", driver="recorded", agent_version=VERSION) + + +def test_an_unwitnessed_version_raises(): + with pytest.raises(recorded_driver.NoRecording): + experiment.run_trial(AGENT, "deny", event="pre_tool", driver="recorded", agent_version="0.0.1") + + +def test_an_agent_with_no_recording_has_none(): + assert recorded_driver.has_recording("no-such-agent") is False + + +def test_as_report_marks_the_recorded_basis_and_carries_recorded_version(): + results = [experiment.run_trial(AGENT, "deny", event="pre_tool", driver="recorded", agent_version=VERSION)] + report = experiment_report.as_report(results, version=VERSION) + assert report["driver"] == "recorded" + assert report["recorded_version"] == VERSION + # The seed recording covers only pre_tool, not every canonical experiment event. + assert report["basis"] == "live-run-partial" + + +def test_a_recorded_report_claiming_a_newer_version_than_the_recording_is_rejected(): + results = [experiment.run_trial(AGENT, "deny", event="pre_tool", driver="recorded", agent_version=VERSION)] + with pytest.raises(evidence_report.InvalidReportError, match="exceeds recorded_version"): + experiment_report.as_report(results, version="99.0.0") + + +def test_a_recorded_report_may_still_claim_exactly_the_recorded_version(): + results = [experiment.run_trial(AGENT, "deny", event="pre_tool", driver="recorded", agent_version=VERSION)] + assert experiment_report.as_report(results, version=VERSION) + + +# --- --record ---------------------------------------------------------------------- + + +def test_record_refuses_the_reference_driver(): + with pytest.raises(SystemExit): + experiment.main(["run", "--agent", AGENT, "--driver", "reference", "--record", "--agent-version", "1.0.0"]) + + +def test_record_refuses_the_recorded_driver(): + with pytest.raises(SystemExit): + experiment.main(["run", "--agent", AGENT, "--driver", "recorded", "--record", "--agent-version", "1.0.0"]) + + +def test_record_requires_agent_version(): + with pytest.raises(SystemExit): + experiment.main(["run", "--agent", AGENT, "--driver", "true", "--record"]) + + +def test_record_round_trips_a_synthetic_run(): + """--record writes a file the recorded driver can then replay -- proven end to end.""" + scratch_version = "0.0.1-test" + written = recorded_driver.recordings.path_for(AGENT, scratch_version) + try: + code = experiment.main( + [ + "run", + "--agent", + AGENT, + "--driver", + "true", + "--trial", + "allow", + "--record", + "--agent-version", + scratch_version, + "--json", + ] + ) + assert code == 0 + assert os.path.exists(written) + with open(written, encoding="utf-8") as fh: + body = json.load(fh) + assert body["agent"] == AGENT + assert body["version"] == scratch_version + assert body["driver"] == "real-agent" + assert "allow" in body["events"]["pre_tool"]["trials"] + finally: + if os.path.exists(written): + os.remove(written) + + +def test_record_leaves_no_scratch_workspace_behind(): + scratch_version = "0.0.2-test" + path = None + try: + results = [ + experiment.run_trial( + AGENT, "allow", event="pre_tool", driver="true", keep=True, agent_version=scratch_version + ) + ] + path = recorded_driver.record( + agent=AGENT, version=scratch_version, event="pre_tool", results=results, platform=sys.platform + ) + assert results[0]["workspace"] and os.path.isdir(results[0]["workspace"]) + finally: + if results and results[0]["workspace"]: + shutil.rmtree(results[0]["workspace"], ignore_errors=True) + if path and os.path.exists(path): + os.remove(path) diff --git a/tools/experiment.py b/tools/experiment.py index 9e95d3e..bfb82a0 100644 --- a/tools/experiment.py +++ b/tools/experiment.py @@ -20,9 +20,10 @@ python3 tools/experiment.py run --agent claude_code --event stop python3 tools/experiment.py run --agent claude_code --trial crash --keep -The default driver is tools/reference_agent.py: the vendor's documentation, made -executable. It needs no credentials, so the whole harness runs in CI for free, and a -disagreement between it and the real agent is precisely a documentation bug. +Absent --driver, this replays a recording (tools/recorded_driver.py) if one exists for the +agent, else falls back to tools/reference_agent.py -- the vendor's documentation, made +executable. Both need no credentials, so the whole harness runs in CI for free, and a +disagreement between either and the real agent is a finding. SAFETY: this probe denies, crashes and stalls on purpose. Every trial runs in a fresh temporary directory that this module creates and removes, with a config written only @@ -46,6 +47,7 @@ import experiment_probe # noqa: E402 import experiment_report # noqa: E402 +import recorded_driver # noqa: E402 import reference_agent # noqa: E402 from agentseam import adapters, contract # noqa: E402 @@ -155,10 +157,14 @@ def _classify(trial, event, observed, invocations): return field, (when_blocked if blocked else when_ran), reading -def run_trial(agent, trial, *, event=contract.PRE_TOOL, driver="reference", keep=False, timeout=None): - """One trial, start to finish, in a workspace created and destroyed here.""" +def run_trial( + agent, trial, *, event=contract.PRE_TOOL, driver="reference", keep=False, timeout=None, agent_version=None +): + """One trial: a real workspace, or (driver="recorded") a frozen recording replayed.""" if event not in EVENTS: raise ValueError("cannot gate at %r (have: %s)" % (event, ", ".join(EVENTS))) + if driver == recorded_driver.DRIVER_NAME: + return recorded_driver.run_trial(agent, trial, event=event, version=agent_version) adapter = adapters.get(agent) workspace = tempfile.mkdtemp(prefix="agentseam-exp-%s-%s-" % (agent, trial)) record_dir = os.path.join(workspace, ".record") @@ -250,7 +256,7 @@ def main(argv=None): run.add_argument( "--event", default=contract.PRE_TOOL, choices=EVENTS, help="which gate to wire the probe at (default: pre_tool)" ) - run.add_argument("--driver", default="reference", help="'reference', or a shell template containing {prompt}") + recorded_driver.add_cli_args(run) run.add_argument("--keep", action="store_true", help="leave the scratch workspace for inspection") run.add_argument("--json", action="store_true") run.add_argument("--report", action="store_true", help="emit a submittable evidence report") @@ -263,8 +269,18 @@ def main(argv=None): print("%-10s %s" % (name, what)) return 0 + args.driver = recorded_driver.resolve_driver(args.agent, args.driver) + if args.record: + recorded_driver.check_record_args(parser, driver=args.driver, agent_version=args.agent_version) + trials = args.trial or sorted(experiment_probe.BEHAVIOURS) - results = [run_trial(args.agent, t, event=args.event, driver=args.driver, keep=args.keep) for t in trials] + keep = args.keep or args.record + results = [ + run_trial(args.agent, t, event=args.event, driver=args.driver, keep=keep, agent_version=args.agent_version) + for t in trials + ] + if args.record: + recorded_driver.finalize_record(args, results) if args.report: report = experiment_report.as_report(results, version=args.agent_version, reporter=args.reporter) print(json.dumps(report, indent=2)) diff --git a/tools/experiment_report.py b/tools/experiment_report.py index c565f8f..fe273b9 100644 --- a/tools/experiment_report.py +++ b/tools/experiment_report.py @@ -63,25 +63,49 @@ def as_report(results, *, version=None, reporter=None, notes=None, today=None): The basis is derived from the driver, never chosen by the caller: a run against the reference is documentation and says so. evidence_report.validate() enforces the same - rule independently, so a hand-edited report cannot claim more than it earned. + rule independently, so a hand-edited report cannot claim more than it earned. A run + against the recorded driver reports what the *recording* witnessed, not a fresh basis: + it carries `recorded_version` so evidence_report.validate() can refuse a `version` the + recording never ran against. """ driver = results[0]["driver"] measured = {} for r in results: measured.update(r["measured"]) - report = { + if driver == evidence_report.RECORDED_DRIVER: + report = _recorded_report(results, measured, today=today) + else: + report = { + "report_version": evidence_report.REPORT_VERSION, + "agent": results[0]["agent"], + "basis": "vendor-docs" if driver == evidence_report.REFERENCE_DRIVER else "live-run-partial", + "date": (today or date.today()).isoformat(), + "driver": "reference" if driver == evidence_report.REFERENCE_DRIVER else "real-agent", + "experiments": measured, + "platform": sys.platform, + } + for key, value in (("version", version), ("reporter", reporter), ("notes", notes)): + if value: + report[key] = value + return evidence_report.validate(report) + + +def _recorded_report(results, measured, *, today=None): + """The report shape for a replayed recording: same basis family the recording earned.""" + outcome = results[0]["outcome"] + import experiment # local: avoids a load-time cycle with experiment.py's own import of this module + + fully_covered = set(outcome.get("recorded_events") or ()) >= set(experiment.EVENTS) + return { "report_version": evidence_report.REPORT_VERSION, "agent": results[0]["agent"], - "basis": "vendor-docs" if driver == evidence_report.REFERENCE_DRIVER else "live-run-partial", + "basis": "live-run" if fully_covered else "live-run-partial", "date": (today or date.today()).isoformat(), - "driver": "reference" if driver == evidence_report.REFERENCE_DRIVER else "real-agent", + "driver": evidence_report.RECORDED_DRIVER, + "recorded_version": outcome["recorded_version"], "experiments": measured, "platform": sys.platform, } - for key, value in (("version", version), ("reporter", reporter), ("notes", notes)): - if value: - report[key] = value - return evidence_report.validate(report) def render(results, *, agent, event, driver): diff --git a/tools/recorded_driver.py b/tools/recorded_driver.py new file mode 100644 index 0000000..b328340 --- /dev/null +++ b/tools/recorded_driver.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Replays a frozen recording instead of driving a real agent process. + +tools/experiment.py answers "what did the vendor do" by launching a real process. This +answers the same question from a `data/recordings/@.json` file instead -- +the freeze the owner's witness cycle calls for: watch what the vendor was seen to do once, +then test against that freeze until the vendor ships again (tools/watch_versions.py is what +notices the ship). + +A replayed trial hands the recording's own sentinel and hook-invocation counts to +`experiment._classify` -- the exact function a live run would have gone through -- so a +recorded result is not a shortcut that happens to agree with the harness, it is the harness, +minus the subprocess. `record()` is the other half: it is what `tools/experiment.py --record` +calls to freeze a real run's results into that same file. Reading a recording back, so the +package and its consumers agree on what one contains, lives in `agentseam.recordings` +instead -- recordings are data the installed, stdlib-only package reads; this module, the +writer and replay driver, is a dev-only tool. +""" + +from __future__ import annotations + +import json +import os +import shutil +import sys +from datetime import date + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..", "src")) + +from agentseam import recordings # noqa: E402 +from agentseam.evidence_report import RECORDED_DRIVER, REFERENCE_DRIVER # noqa: E402 + +#: The driver name tools/experiment.py dispatches on. +DRIVER_NAME = RECORDED_DRIVER + +#: Drivers `--record` refuses: neither ever ran a real agent, so neither can produce live +#: evidence -- the same rule evidence_report.py enforces for a submitted report. +NON_LIVE_DRIVERS = (REFERENCE_DRIVER, DRIVER_NAME) + + +class NoRecording(Exception): + """No recording covers the (agent, version, event, trial) asked for.""" + + +def has_recording(agent): + return recordings.latest_version(agent) is not None + + +def add_cli_args(run_parser): + """--driver and --record, added here since both concern only this module's behaviour.""" + run_parser.add_argument( + "--driver", + default=None, + help="'reference', 'recorded', or a shell template containing {prompt}; " + "default: 'recorded' if a recording exists for --agent, else 'reference'", + ) + run_parser.add_argument( + "--record", action="store_true", help="freeze this run into data/recordings/@.json" + ) + + +def resolve_driver(agent, driver): + """`driver` if given, else 'recorded' when a recording exists for `agent`, else the + reference. Centralised here so tools/experiment.py's CLI stays a thin dispatcher.""" + if driver is not None: + return driver + return DRIVER_NAME if has_recording(agent) else REFERENCE_DRIVER + + +def check_record_args(parser, *, driver, agent_version): + """Exit via `parser.error` if `--record` cannot proceed with these arguments.""" + if driver in NON_LIVE_DRIVERS: + parser.error("--record needs a real agent: %r cannot produce live evidence" % driver) + if not agent_version: + parser.error("--record requires --agent-version") + + +def finalize_record(args, results): + """Write the recording from a completed `run` invocation's own parsed args and results, + then clean up any workspace `--record` asked `run_trial` to keep only for this.""" + path = record( + agent=args.agent, + version=args.agent_version, + event=args.event, + results=results, + platform=sys.platform, + reporter=args.reporter, + ) + print("recorded: %s" % path) + if not args.keep: + for r in results: + if r["workspace"]: + shutil.rmtree(r["workspace"], ignore_errors=True) + return path + + +def _trial_data(agent, trial, event, version): + body = recordings.load_recording(agent, version) + if body is None: + raise NoRecording("no recording for %s%s" % (agent, ("@" + version) if version else "")) + event_body = (body.get("events") or {}).get(event) + if event_body is None: + raise NoRecording("%s@%s never recorded %s" % (agent, body["version"], event)) + trial_body = (event_body.get("trials") or {}).get(trial) + if trial_body is None: + raise NoRecording("%s@%s/%s never recorded the %r trial" % (agent, body["version"], event, trial)) + return body, trial_body + + +def run_trial(agent, trial, *, event, version=None): + """One trial's result, replayed from a recording through the real classifier. + + Shaped exactly like experiment.run_trial's return value, so a caller cannot tell the + difference except by `driver` and `outcome`. No process is launched: the recording + already holds the sentinel counts and hook-invocation count a real run produced. + """ + body, trial_body = _trial_data(agent, trial, event, version) + import experiment # local: experiment.py imports this module, so this stays lazy + + observed = dict(trial_body["observed"]) + invocations = trial_body["hook_invocations"] + field, value, reading = experiment._classify(trial, event, observed, invocations) # noqa: SLF001 + return { + "agent": agent, + "trial": trial, + "event": event, + "driver": DRIVER_NAME, + "measured": {field: value}, + "reading": reading, + "observed": observed, + "hook_invocations": invocations, + "outcome": { + "recorded_version": body["version"], + "recorded_date": body["date"], + "recorded_events": tuple(sorted(body["events"])), + }, + "workspace": None, + } + + +def _invocation_records(result): + workspace = result.get("workspace") + if not workspace: + return [] + record_path = os.path.join(workspace, ".record", "invocations.jsonl") + if not os.path.exists(record_path): + return [] + with open(record_path, encoding="utf-8") as fh: + return [json.loads(line) for line in fh if line.strip()] + + +def record(*, agent, version, event, results, platform, reporter=None, notes=None, today=None): + """Freeze one event's trial results into data/recordings/@.json. + + Appends to an existing recording for the same (agent, version) instead of overwriting + it, so running every gate as separate invocations builds one file -- a partial + recording is simply one whose `events` does not yet name every gate. + """ + existing = recordings.load_recording(agent, version) or { + "agent": agent, + "version": version, + "platform": platform, + "date": (today or date.today()).isoformat(), + "driver": "real-agent", + } + if reporter: + existing["reporter"] = reporter + if notes: + existing["notes"] = notes + existing.setdefault("events", {}) + + trials, keys = {}, set() + for result in results: + trials[result["trial"]] = { + "observed": dict(result["observed"]), + "hook_invocations": result["hook_invocations"], + } + for invocation in _invocation_records(result): + keys.update(invocation.get("keys") or ()) + existing["events"][event] = {"trials": trials, "payload_keys": sorted(keys)} + + path = recordings.path_for(agent, version) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + json.dump(existing, fh, indent=2, sort_keys=True) + fh.write("\n") + return path From e819f67c762fca08ccebec3a4b1c73be3243d79a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:24:22 +0000 Subject: [PATCH 19/29] cursor: respond infers an unnamed event the same way parse does Finding raw[2].findings[7] of the vendor-truth review, reproduced by execution. parse() infers afterFileEdit from an edits[] list; respond() re-derived the event from event.tool, which parse() fills from tool_name when the payload carries one, and so fell through to the entry's default beforeShellExecution gate. An unnamed edits payload with a tool_name therefore parsed as file_changed and was answered with a permission verdict at an event that reads no output -- an already-landed write reported as prevented. respond() now calls cursor_wire() itself. event.tool is read only for an Event carrying no payload, where parse() left the inferred name there and there is nothing to re-infer from. No frozen wire output moves: every golden scenario names its own event. Signed-off-by: Claude Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 10 ++++++++++ src/agentseam/adapters/_cursor.py | 9 +++++++-- tests/test_adapter_cursor.py | 8 ++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebbb463..e30c7a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,16 @@ versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html). re-run -- still show their original drift. ### Fixed +- **Cursor answers an unnamed payload at the event `parse()` said it was** (`adapters/ + _cursor.py`; vendor-truth review finding `raw[2].findings[7]`). `parse()` infers + `afterFileEdit` from an `edits[]` list, but `respond()` re-derived the event from + `event.tool` -- which `parse()` fills from `tool_name` when the payload carries one -- and + fell through to the entry's default `beforeShellExecution` gate. Reproduced: an unnamed + `edits` payload with a `tool_name` parsed as `file_changed` and was answered with + `{"permission": "deny"}`, a permission verdict at an event documented as reading no output + -- the fake-gate half of bug class 2, reporting an already-landed write as prevented. + `respond()` now calls the same `cursor_wire()` inference `parse()` uses, so the two cannot + diverge. No frozen wire output moves: every golden scenario names its event. - **The stop gate's block observable is the hook re-firing, not a second action run** (`tools/experiment.py`). `_blocked()` used to score a Stop-gate block by reading the sentinel twice, on the theory that an agent refused permission to finish comes back diff --git a/src/agentseam/adapters/_cursor.py b/src/agentseam/adapters/_cursor.py index bbf0186..252617e 100644 --- a/src/agentseam/adapters/_cursor.py +++ b/src/agentseam/adapters/_cursor.py @@ -73,11 +73,16 @@ def _because(reason, note): def _wire_of(cfg, event): - """The wire name to answer at: the payload's own, `tool` where `parse` kept it there, - else the entry's default gate.""" + """The wire name to answer at -- `cursor_wire` again, so respond and parse cannot diverge. + + `tool` is read only for an Event carrying no payload, where `parse` left the inferred + name there; without a payload there is nothing to re-infer from. + """ name = (event.raw or {}).get("hook_event_name") if name in cfg["events"]: return name + if event.raw: + return cursor_wire(event.raw) return event.tool if event.tool in cfg["events"] else cfg["verdicts"].get("default_wire_event") diff --git a/tests/test_adapter_cursor.py b/tests/test_adapter_cursor.py index 920b691..6a81d4e 100644 --- a/tests/test_adapter_cursor.py +++ b/tests/test_adapter_cursor.py @@ -132,3 +132,11 @@ def test_every_gate_is_installed_fail_closed_including_the_prompt_one(): assert cfg["hooks"]["beforeSubmitPrompt"][0]["failClosed"] is True assert cfg["hooks"]["preToolUse"][0]["failClosed"] is True assert "failClosed" not in cfg["hooks"]["postToolUse"][0] + + +def test_respond_infers_the_unnamed_event_the_same_way_parse_does(): + """An unnamed edits payload carrying tool_name parsed as file_changed and was answered""" + raw = {"conversation_id": "c1", "file_path": "/repo/a.py", "edits": [{"new_string": "s"}], "tool_name": "Edit"} + mod = A.adapters.get("cursor") + assert mod.parse(raw).event == A.FILE_CHANGED + assert mod.respond(Decision.deny("no"), mod.parse(raw)) == ("", 0) From f9360ad605b1a0e9298208a875d0480ec0e15c07 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:24:32 +0000 Subject: [PATCH 20/29] matrix: point claude_code pre_tool evidence at its recording The six pre_tool per-claim records repeated the same live-run prose six times; each now points test at data/recordings/claude_code@2.1.263.json instead, so the basis chain is claim -> recording -> live run, version-pinned. Same basis/date/version, just no longer copy-pasted. Co-Authored-By: Claude Sonnet 5 Signed-off-by: Claude --- src/agentseam/data/matrix.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/agentseam/data/matrix.json b/src/agentseam/data/matrix.json index f0b4446..a4a105a 100644 --- a/src/agentseam/data/matrix.json +++ b/src/agentseam/data/matrix.json @@ -24,37 +24,37 @@ "basis": "live-run", "date": "2026-09-07", "version": "2.1.263", - "method": "real Claude Code CLI 2.1.263 (Linux) via tools/experiment.py --driver \"claude -p {prompt} --permission-mode acceptEdits\", run twice with identical results" + "test": "data/recordings/claude_code@2.1.263.json" }, "rewrite": { "basis": "live-run", "date": "2026-09-07", "version": "2.1.263", - "method": "real Claude Code CLI 2.1.263 (Linux) via tools/experiment.py --driver \"claude -p {prompt} --permission-mode acceptEdits\", run twice with identical results" + "test": "data/recordings/claude_code@2.1.263.json" }, "fail_mode": { "basis": "live-run", "date": "2026-09-07", "version": "2.1.263", - "method": "real Claude Code CLI 2.1.263 (Linux) via tools/experiment.py --driver \"claude -p {prompt} --permission-mode acceptEdits\", run twice with identical results" + "test": "data/recordings/claude_code@2.1.263.json" }, "silence_means": { "basis": "live-run", "date": "2026-09-07", "version": "2.1.263", - "method": "real Claude Code CLI 2.1.263 (Linux) via tools/experiment.py --driver \"claude -p {prompt} --permission-mode acceptEdits\", run twice with identical results" + "test": "data/recordings/claude_code@2.1.263.json" }, "timeout_fail_mode": { "basis": "live-run", "date": "2026-09-07", "version": "2.1.263", - "method": "real Claude Code CLI 2.1.263 (Linux) via tools/experiment.py --driver \"claude -p {prompt} --permission-mode acceptEdits\", run twice with identical results" + "test": "data/recordings/claude_code@2.1.263.json" }, "unknown_verb_means": { "basis": "live-run", "date": "2026-09-07", "version": "2.1.263", - "method": "real Claude Code CLI 2.1.263 (Linux) via tools/experiment.py --driver \"claude -p {prompt} --permission-mode acceptEdits\", run twice with identical results" + "test": "data/recordings/claude_code@2.1.263.json" } }, "silence_means": "allow", From 2253fe8fe75448d52a06ea038653a1859998254d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:24:45 +0000 Subject: [PATCH 21/29] watch_versions: one drift issue per agent, self-sufficient for a stranger A drift issue is a work order for whoever has the agent installed, not a note to the maintainer: watch_versions.py --open-issues now opens one issue per drifted agent instead of one combined issue, titled "evidence: -- re-witness wanted", labelled evidence and help wanted (created if absent), and skips an agent that already has one open. The body carries the exact --record commands for the gates the kit can actually gate at, what running them produces, and both ways to submit the result. A covered gate now compares against its recording's own version ahead of the per-claim override. CONTRIBUTING.md points at the same steps; agentseam matrix --evidence shows the recorded version beside the row's. Co-Authored-By: Claude Sonnet 5 Signed-off-by: Claude --- .github/workflows/staleness.yml | 24 ++--- CONTRIBUTING.md | 20 +++++ src/agentseam/cli.py | 10 ++- tests/test_cli.py | 8 ++ tests/test_watch_versions.py | 107 ++++++++++++++++++++++ tools/watch_versions.py | 154 ++++++++++++++++++++++++++++---- 6 files changed, 285 insertions(+), 38 deletions(-) create mode 100644 tests/test_watch_versions.py diff --git a/.github/workflows/staleness.yml b/.github/workflows/staleness.yml index a0f744d..2922e2d 100644 --- a/.github/workflows/staleness.yml +++ b/.github/workflows/staleness.yml @@ -12,12 +12,15 @@ on: - cron: "17 6 * * 1" workflow_dispatch: pull_request: - # On PRs it runs read-only: a change to matrix-evidence.json should be able to show - # its effect on the table without waiting a week to find out. + # On PRs it runs read-only: a change to matrix.json's evidence or a new recording + # should be able to show its effect on the table without waiting a week to find out. paths: - - "src/agentseam/data/matrix-evidence.json" + - "src/agentseam/data/matrix.json" + - "src/agentseam/data/recordings/*.json" - "src/agentseam/data/vendor-releases.json" + - "src/agentseam/recordings.py" - "src/agentseam/staleness.py" + - "tools/recorded_driver.py" - "tools/watch_versions.py" permissions: @@ -78,17 +81,8 @@ jobs: with: python-version: "3.13" - - name: Open an issue when a row's agent has moved + - name: Open one issue per drifted agent (skips an agent that already has one open) env: GH_TOKEN: ${{ github.token }} - run: | - if python3 tools/watch_versions.py --no-fail-on-drift --json \ - | python3 -c 'import json,sys; sys.exit(0 if [r for r in json.load(sys.stdin) if r["verdict"]=="drifted"] else 1)'; then - python3 tools/watch_versions.py --issue-body --no-fail-on-drift > body.md - gh issue create \ - --title "Evidence drift: an agent has shipped since its row was verified" \ - --body-file body.md \ - --label evidence - else - echo "no drift" - fi + GH_REPO: ${{ github.repository }} + run: python3 tools/watch_versions.py --open-issues --no-fail-on-drift diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b423430..4f85c10 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -91,6 +91,26 @@ Evidence carries the reporter's handle. Age and version drift are displayed rath hidden — see `agentseam matrix --evidence`. A row that says "verified against 3.17.8, 87 days ago" is more useful than one that silently implies it is current. +### When a drift issue names your agent + +A weekly check (`tools/watch_versions.py`) opens an issue titled `evidence: + — re-witness wanted` when an agent has shipped since its evidence was taken. That +issue is a work order for whoever has the agent installed — the same two-minute loop above, +`--record` added: + +```bash +python3 tools/experiment.py run --agent --event \ + --driver "" \ + --agent-version --record --reporter @yourhandle +``` + +This freezes what you saw into `data/recordings/@.json`, which +`--driver recorded` then replays in CI instead of anyone re-running a real agent every +time. Submit it the same two ways as any other evidence: a PR adding that file (plus the +`data/matrix.json` per-claim `test` pointers it backs), or the **Evidence report** template +with the recording pasted in. A run against `reference` or `recorded` never counts as the +re-witness the issue is asking for — only a real agent does. + ## Local checks ```bash diff --git a/src/agentseam/cli.py b/src/agentseam/cli.py index 1d741a6..ac8252c 100644 --- a/src/agentseam/cli.py +++ b/src/agentseam/cli.py @@ -10,7 +10,7 @@ import textwrap from datetime import date -from . import __version__, adapters +from . import __version__, adapters, recordings from . import install as install_mod from . import instructions as instructions_mod from . import packaging as packaging_mod @@ -50,21 +50,23 @@ def _print_evidence(): and the provenance of its cells are the same claim, and showing one without the other is what lets a documentation guess pass for a measurement. """ - print("%-16s %-20s %-12s %-11s %s" % ("agent", "basis", "version", "verdict", "date")) + print("%-16s %-20s %-12s %-11s %-12s %s" % ("agent", "basis", "version", "verdict", "recorded", "date")) for name in sorted(MATRIX): verified = MATRIX[name]["verified"] state = staleness_mod.status(verified) print( - "%-16s %-20s %-12s %-11s %s" + "%-16s %-20s %-12s %-11s %-12s %s" % ( name, verified.get("basis", "-"), str(verified.get("version", "-"))[:12], state["verdict"], + recordings.latest_version(name) or "-", verified.get("date", "-"), ) ) - print("\nverdicts: fresh (compared, current) | unchecked (no comparison was made)") + print("\nrecorded: the newest data/recordings/@.json, or '-' if never witnessed") + print("verdicts: fresh (compared, current) | unchecked (no comparison was made)") print( " stale (older than %d days) | unmeasured (never touched a running agent)" % staleness_mod.STALE_AFTER_DAYS diff --git a/tests/test_cli.py b/tests/test_cli.py index 8053d5c..a93e183 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -38,6 +38,14 @@ def test_matrix_renders(): assert "claude_code" in out.stdout and "best-effort" in out.stdout +def test_matrix_evidence_shows_the_recorded_version_beside_the_rows(): + out = _run(["matrix", "--evidence"]) + assert out.returncode == 0 + line = next(row for row in out.stdout.splitlines() if row.startswith("claude_code ")) + assert "2.1.247" in line # the row's own verified.version + assert "2.1.263" in line # the newest committed recording + + #: A portable `head -3`: read three lines, then exit and drop the read end of the pipe. #: Spawned rather than shelled out to because Windows has no `head`. _HEAD_3 = "import sys\nfor _ in range(3): sys.stdin.readline()\n" diff --git a/tests/test_watch_versions.py b/tests/test_watch_versions.py new file mode 100644 index 0000000..f2cc77d --- /dev/null +++ b/tests/test_watch_versions.py @@ -0,0 +1,107 @@ +"""The staleness watcher's per-agent drift issue: grouping, title, and self-sufficient body. + +`check()`'s own network-touching path is exercised by test_staleness.py through the pure +`staleness` module; this file is about what happens once a row has already drifted -- +grouping rows into one issue per agent, and the body a stranger with the agent installed +needs with nothing else. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +TOOLS = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "tools") +sys.path.insert(0, TOOLS) + +import watch_versions # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] + +FIXTURE_ROWS = [ + { + "agent": "claude_code", + "event": "pre_tool", + "verdict": "drifted", + "recorded_version": "2.1.247", + "current_version": "2.1.263", + "age_days": 10, + }, + { + "agent": "claude_code", + "event": "stop", + "verdict": "drifted", + "recorded_version": "2.1.247", + "current_version": "2.1.263", + "age_days": 10, + }, + { + "agent": "cursor", + "event": None, + "verdict": "unmeasured", + "recorded_version": "3.17.0", + "current_version": "3.18.0", + "age_days": 5, + }, +] + + +def test_agents_needing_issues_groups_by_agent_and_skips_the_undrifted(): + grouped = watch_versions.agents_needing_issues(FIXTURE_ROWS) + assert set(grouped) == {"claude_code"} + assert len(grouped["claude_code"]) == 2 + + +def test_issue_title_names_the_agent_and_the_new_version(): + title = watch_versions.issue_title("claude_code", "2.1.263") + assert title == "evidence: claude_code 2.1.263 — re-witness wanted" + + +def test_issue_body_is_self_sufficient_for_a_stranger(): + """The verify command in the brief: agent, both versions, the --record command, both + submission paths, and the honesty rule -- all present with nothing else to look up.""" + body = watch_versions.issue_body_for_agent( + "claude_code", watch_versions.agents_needing_issues(FIXTURE_ROWS)["claude_code"] + ) + assert "claude_code" in body + assert "2.1.247" in body + assert "2.1.263" in body + assert "--record" in body + assert "pip install agentseam" in body + assert "open a PR" in body + assert "Evidence report" in body + assert "never counts as a re-witness" in body + + +def test_issue_body_only_suggests_gates_the_kit_can_gate_at(): + """claude_code's row claims post_tool, session_start, etc. -- experiment.py cannot wire + a probe at any of those, so the suggested command must never name one.""" + body = watch_versions.issue_body_for_agent( + "claude_code", watch_versions.agents_needing_issues(FIXTURE_ROWS)["claude_code"] + ) + for event in ("pre_tool", "prompt_submit", "stop"): + assert ("--event %s" % event) in body + assert "--event post_tool" not in body + + +def test_issue_body_suggests_one_command_per_gate_not_a_repeated_flag(): + """tools/experiment.py's --event takes a single value; a comma- or repeat-flag command + would silently only run the last gate.""" + body = watch_versions.issue_body_for_agent( + "claude_code", watch_versions.agents_needing_issues(FIXTURE_ROWS)["claude_code"] + ) + assert body.count("tools/experiment.py run") >= 2 + assert "--event pre_tool --event" not in body + + +def test_workflow_delegates_to_open_issues(): + """act-free assertion on the YAML: the report job is one call to --open-issues, which + is unit-tested above for one-issue-per-agent and (via ISSUE_LABELS) both labels.""" + text = (ROOT / ".github" / "workflows" / "staleness.yml").read_text(encoding="utf-8") + assert "--open-issues" in text + assert "issues: write" in text + + +def test_open_issues_carries_both_labels(): + assert set(watch_versions.ISSUE_LABELS) == {"evidence", "help wanted"} diff --git a/tools/watch_versions.py b/tools/watch_versions.py index 70edf39..f3f82fd 100644 --- a/tools/watch_versions.py +++ b/tools/watch_versions.py @@ -23,6 +23,7 @@ import argparse import json import os +import subprocess import sys import urllib.error import urllib.request @@ -30,12 +31,21 @@ HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.join(HERE, "..", "src")) -from agentseam import staleness # noqa: E402 +from agentseam import recordings, staleness # noqa: E402 from agentseam._data import load # noqa: E402 +from agentseam.contract import PRE_TOOL, PROMPT_SUBMIT, STOP # noqa: E402 from agentseam.matrix_data import MATRIX # noqa: E402 from agentseam.matrix_evidence import EVIDENCE, claim_record # noqa: E402 from agentseam.matrix_terms import CLAIM_FAIL_MODE # noqa: E402 +#: gh CLI labels a drift issue always carries. Created with --force if the repo lacks them. +ISSUE_LABELS = ("evidence", "help wanted") +_LABEL_COLORS = {"evidence": "0e8a16", "help wanted": "128a0c"} + +#: Gates tools/experiment.py can actually wire a probe at (mirrors its own EVENTS) -- a +#: drift issue must never suggest re-witnessing a gate the kit cannot even gate at. +EXPERIMENTABLE_EVENTS = (PROMPT_SUBMIT, PRE_TOOL, STOP) + SOURCES = load("vendor-releases.json") TIMEOUT_SECONDS = 20 USER_AGENT = "agentseam-version-watch" @@ -70,12 +80,18 @@ def latest_version(source): def _event_versions(agent): - """(event, record) for every event this agent's row claims, reading the `fail_mode` - claim's own evidence where it names a version -- so a per-claim override (task 4) is - what drift compares against, and the row's record is the fallback everywhere else.""" + """(event, record) for every event this agent's row claims. A gate a committed + recording covers compares against *that* recording's version -- the frozen artifact is + now the ground truth for it, ahead of the per-claim override it otherwise falls back to + (task 4), which itself falls back to the row's own record.""" row = MATRIX.get(agent) or {} + body = recordings.load_recording(agent) + covered = set((body or {}).get("events") or ()) out = [] for event in row.get("events", {}): + if event in covered: + out.append((event, {"basis": "live-run", "version": body["version"], "date": body["date"]})) + continue record = claim_record(row, event, CLAIM_FAIL_MODE) out.append((event, record if record.get("version") else EVIDENCE.get(agent, {}))) return out @@ -115,38 +131,138 @@ def _label(r): return r["agent"] if r["event"] is None else "%s/%s" % (r["agent"], r["event"]) -def issue_body(rows): - """Markdown for the issue a scheduled run opens. Names the agent and what moved.""" - lines = ["The following rows describe an agent version that is no longer current.", ""] +def issue_title(agent, current_version): + """The exact title a drift issue carries -- also the key `open_issues` searches by to + decide whether one is already open, so this is the one place that spelling lives.""" + return "evidence: %s %s — re-witness wanted" % (agent, current_version) + + +def agents_needing_issues(rows): + """Drifted agents, each with its own row(s) -- one issue per agent, not per row.""" + by_agent = {} for r in drifted(rows): - lines.append( - "- **%s** — evidence taken against `%s` (%s days ago); `%s` is now published." - % (_label(r), r["recorded_version"], r["age_days"], r["current_version"]) + by_agent.setdefault(r["agent"], []).append(r) + return dict(sorted(by_agent.items())) + + +def issue_body_for_agent(agent, agent_rows): + """Self-sufficient markdown for one drifted agent: what moved, the exact commands to + re-witness it, and both ways to submit the result. + + A drift issue is a work order for whoever has `agent` installed, not a note to the + maintainer (owner decision 2026-09-07, org-plan plan/agentseam-project.md 'Evidence + layer') -- so this owes the reader everything, not a pointer to CONTRIBUTING.md. + """ + current = agent_rows[0]["current_version"] + moved = "\n".join( + "- `%s`: evidence taken against `%s` (%s days ago); `%s` is now published." + % (r["event"] or agent, r["recorded_version"], r["age_days"], r["current_version"]) + for r in agent_rows + ) + row_events = set((MATRIX.get(agent) or {}).get("events") or {}) + events = sorted(row_events & set(EXPERIMENTABLE_EVENTS)) or [PRE_TOOL] + # One invocation per gate, not one command with a repeated --event: the CLI takes a + # single --event, and --record appending to the same file is what makes that build one + # recording covering every gate rather than a race to overwrite it. + per_event_commands = "\n".join( + "python3 tools/experiment.py run --agent %s --event %s \\\n" + ' --driver "" \\\n' + " --agent-version %s --record --reporter @yourhandle" % (agent, event, current) + for event in events + ) + return "\n".join( + [ + "`%s` has moved since its evidence was taken:" % agent, + "", + moved, + "", + "## Re-witness it", + "", + "You need `%s` installed and runnable headlessly from a terminal." % agent, + "", + "```bash", + "pip install agentseam", + per_event_commands, + "```", + "", + "This writes `data/recordings/%s@%s.json` and touches nothing else on your machine." % (agent, current), + "", + "**A run against the `reference` or `recorded` driver never counts as a re-witness** --", + "only a real, running `%s` does." % agent, + "", + "## Submit it", + "", + "- **Preferred**: open a PR adding that recording file plus the updated", + " `data/matrix.json` per-claim `test` pointers (see any recorded row for the shape).", + '- **Or**: open an "Evidence report" issue and paste the recording JSON for a', + " maintainer to land.", + ] + ) + + +def _ensure_labels(): + for label in ISSUE_LABELS: + subprocess.run( # noqa: S603, S607 + ["gh", "label", "create", label, "--color", _LABEL_COLORS[label], "--force"], + check=False, + capture_output=True, + text=True, ) - lines += [ - "", - "This does not mean the row is wrong. It means nothing in this repository can", - "currently say whether it is still right. Re-run `tools/experiment.py` against the", - "new version, or mark the row re-verified if the hook contract is unchanged.", - ] - return "\n".join(lines) + + +def _open_issue_titles(): + result = subprocess.run( # noqa: S603, S607 + ["gh", "issue", "list", "--state", "open", "--limit", "200", "--json", "title"], + check=True, + capture_output=True, + text=True, + ) + return {row["title"] for row in json.loads(result.stdout)} + + +def open_issues(rows): + """Open one GitHub issue per drifted agent lacking one already open. Returns the titles + created, so a caller (or a test) can see exactly what happened without re-querying gh.""" + _ensure_labels() + existing = _open_issue_titles() + created = [] + for agent, agent_rows in agents_needing_issues(rows).items(): + title = issue_title(agent, agent_rows[0]["current_version"]) + if title in existing: + continue + command = ["gh", "issue", "create", "--title", title, "--body", issue_body_for_agent(agent, agent_rows)] + for label in ISSUE_LABELS: + command += ["--label", label] + subprocess.run(command, check=True) # noqa: S603 + created.append(title) + return created def main(argv=None): parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--agent", action="append", help="repeatable; default is every agent") parser.add_argument("--json", action="store_true") - parser.add_argument("--issue-body", action="store_true", help="markdown for a drift issue") + parser.add_argument("--issue-body", action="store_true", help="markdown for a drift issue, one per drifted agent") + parser.add_argument( + "--open-issues", action="store_true", help="open one GitHub issue per drifted agent via gh (needs GH_TOKEN)" + ) parser.add_argument("--fail-on-drift", action="store_true", default=True) parser.add_argument("--no-fail-on-drift", dest="fail_on_drift", action="store_false") args = parser.parse_args(argv) rows = check(args.agent) + if args.open_issues: + for title in open_issues(rows): + print("opened: %s" % title) + return 0 + if args.json: print(json.dumps(rows, indent=2, sort_keys=True)) elif args.issue_body: - print(issue_body(rows)) + for agent, agent_rows in agents_needing_issues(rows).items(): + print(issue_body_for_agent(agent, agent_rows)) + print() else: print("%-24s %-11s %-12s %-12s %-9s %s" % ("agent", "verdict", "recorded", "current", "age", "note")) for r in rows: From 98ae41ea52051ea9e7c7ca1bf5c6df6099aa51c0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:24:51 +0000 Subject: [PATCH 22/29] changelog: recorded driver and the per-agent witness-cycle closure Co-Authored-By: Claude Sonnet 5 Signed-off-by: Claude --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebbb463..d5bdb5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,32 @@ versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ### Added +- **Recorded driver: freeze a witnessed run, replay it in CI** (owner decision 2026-09-07, + org-plan plan/agentseam-project.md "Evidence layer"). The witness cycle: freeze what a + real agent was seen to do, test against the freeze, re-witness only when the vendor + ships -- `tools/watch_versions.py` already detects the ship; nothing froze or replayed + until now. + - `src/agentseam/data/recordings/@.json` (schema alongside it): one file + per witnessed (agent, version), holding the sentinel and hook-invocation counts each + trial produced against a real agent. Immutable once committed -- a newer version is a + new file. `src/agentseam/recordings.py` is the package-side reader (recordings are data + the installed package reads); `tools/recorded_driver.py` is the dev-only writer/replayer. + - `tools/experiment.py run --record` freezes a real-agent run into that file, refusing the + `reference` and `recorded` drivers with the same honesty rule `evidence_report.py` + already enforces on a submitted report. + - `--driver recorded` (the default once a recording exists for `--agent`) replays a + recording through the exact same `_classify` a live run would have used, with no + process launched -- the seven trials against `claude_code@2.1.263` reproduce the + witnessed table in under a second. `evidence_report.py` gains `recorded_version` and + rejects a recorded-driver report claiming a newer `version` than it replayed. + `claude_code`'s `pre_tool` per-claim evidence now points `test` at the recording + instead of repeating its prose six times, so the basis chain is claim -> recording -> + live run. + - `tools/watch_versions.py` compares a covered gate against its recording's version ahead + of the row's own, and now opens one drift issue per agent (was: one combined issue), + self-sufficient for a stranger with the agent installed -- the exact `--record` command, + what it produces, and both ways to submit it. `agentseam matrix --evidence` shows the + recorded version beside the row's. - **Per-claim evidence on the capability matrix, and grading capped by basis** (owner decision 2026-09-01, org-plan plan/agentseam-project.md). Additive data shape: - Every asserted matrix cell field (`block`, `rewrite`, `fail_mode`) now carries its own From da451c1aed4ec7dcd164a479fe2dbd4c736b4d9d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 19:25:01 -0400 Subject: [PATCH 23/29] changelog: seven Windows-only test failures, and the POSIX assumption behind each Signed-off-by: Claude Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebbb463..94761e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,30 @@ versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html). re-run -- still show their original drift. ### Fixed +- **Seven tests that could not pass on Windows, each resting on a POSIX assumption.** All + seven fail on a clean checkout; none was skipped or deleted. + - Five (`test_install.py` x3, `test_adapter_kimi_code.py` x2) set `HOME` by hand and then + asserted against a path derived from it. Only `posixpath.expanduser` reads `HOME`; + `ntpath` reads `USERPROFILE`, then `HOMEDRIVE` + `HOMEPATH`. `conftest.py`'s autouse + `isolated_home` fixture already sets all four and says so in its own docstring -- the + tests now take it as a parameter instead of re-deriving home from one variable. + - `test_examples.py` compared committed pages against freshly built ones with a bare + `open()`. The pages carry em dashes, so under cp1252 every one of the twelve read back + mangled and reported stale, with no way to make it pass: regenerating wrote the same + bytes it had just failed to read. Both readers now say `encoding="utf-8"` -- the + test's, and `examples/generate.py`'s `--check` path, which the pre-commit hook and the + `examples` CI job run and which had the identical latent bug. The writer says + `encoding="utf-8", newline=" +"`, so a regeneration on Windows is byte-identical to one + on the CI runner instead of rewriting all thirteen pages with CRLF. + - `test_git_hooks.py::test_hook_is_executable` read `st_mode & 0o111` from the working + tree. Windows has no POSIX execute bit, so that is 0 for every file -- including one git + records as `100755` and checks out executable elsewhere. It now asserts on the mode git + records, which is what travels with a clone and what decides whether the hook runs. + - Found while fixing those: `test_a_query_never_raises_on_an_undecodable_toml_config` + carried the same `HOME` assumption but *passed* on Windows, because `installed()` + returned `False` for a file that was not on the path it reads. It asserted its own + setup rather than the TOML branch it names. Same fix. - **The stop gate's block observable is the hook re-firing, not a second action run** (`tools/experiment.py`). `_blocked()` used to score a Stop-gate block by reading the sentinel twice, on the theory that an agent refused permission to finish comes back From 2d320fcc05bb62e56f365bde87a4264590c70338 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:25:38 +0000 Subject: [PATCH 24/29] cursor: a rewrite with no input is not the gate failing to express one Finding raw[2].findings[4] of the vendor-truth review, reproduced by execution. Decision.rewrite(None, ...) at preToolUse was refused with "input requires modification, which this gate cannot express" -- untrue at the one Cursor gate that can, and the adapter's headline capability. The handler had simply supplied no replacement, so an operator reading that message files a capability bug against the wrong layer. The engine already separates the two cases with transform_missing_input; _cursor.py now consults it and keeps the "cannot express" wording for the gates where it is true. Pulling the reason out of _gate_payload into _refusal_reason is what makes that branch expressible. The deny itself, which is the safe outcome, is unchanged; one frozen wire byte moves. Signed-off-by: Claude Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 10 +++++++ src/agentseam/adapters/_cursor.py | 37 ++++++++++++++++---------- src/agentseam/data/vendors/cursor.json | 3 ++- tests/fixtures/golden/cursor.json | 2 +- tests/test_adapter_cursor.py | 6 +++++ tools/recount/tables.py | 1 + 6 files changed, 43 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e30c7a2..1ac3490 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,16 @@ versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html). re-run -- still show their original drift. ### Fixed +- **A rewrite with no replacement input is no longer blamed on Cursor's rewrite gate** + (`adapters/_cursor.py`, `data/vendors/cursor.json`; vendor-truth review finding + `raw[2].findings[4]`). `Decision.rewrite(None, ...)` at `preToolUse` was refused with + "input requires modification, which this gate cannot express" -- untrue at the one Cursor + gate that *can* express a rewrite, and the adapter's headline capability. The handler + simply supplied no replacement. The engine already distinguishes the two cases + (`transform_missing_input`); `_cursor.py` now consults it, keeping the "cannot express" + wording for the gates where it is true. One byte of frozen wire output moves + (`rewrite-without-input` at `pre_tool`); the deny itself, which is the safe outcome, is + unchanged. - **Cursor answers an unnamed payload at the event `parse()` said it was** (`adapters/ _cursor.py`; vendor-truth review finding `raw[2].findings[7]`). `parse()` infers `afterFileEdit` from an `edits[]` list, but `respond()` re-derived the event from diff --git a/src/agentseam/adapters/_cursor.py b/src/agentseam/adapters/_cursor.py index 252617e..6868af2 100644 --- a/src/agentseam/adapters/_cursor.py +++ b/src/agentseam/adapters/_cursor.py @@ -19,7 +19,7 @@ TRANSFORM, degraded_from, ) -from ._hook_json import _ESCALATE_FROM_TRANSFORM +from ._hook_json import _ESCALATE_FROM_TRANSFORM, _TRANSFORM_MISSING_INPUT from ._payload import hj_parse #: Wire names other vendors also spell this way; a payload naming one is claimed only on @@ -101,21 +101,30 @@ def _prompt_submit_payload(decision): return _json.dumps(payload), 0 -def _gate_payload(words, notes, gate, decision, name): - """The PRE_TOOL gate's (permission, reason) pair, before the shared trailing message rule.""" - reason = decision.reason +def _refusal_reason(v, gate, decision, name): + """The handler's own reason, plus why the outcome changed shape on the way out.""" + notes = v["degrade_notes"] if decision.outcome == TRANSFORM: - if gate["honours_transform"] and decision.updated_input is not None: - return {"permission": words["allow"], "updated_input": decision.updated_input}, reason - return {"permission": words["block"]}, _because(reason, notes["transform"]) - if decision.outcome == DENY: - return {"permission": words["block"]}, reason + # A gate that DOES honour transform refused only for want of a replacement input; + # "this gate cannot express it" would be false at the one gate that can. + key = _TRANSFORM_MISSING_INPUT if gate["honours_transform"] else "transform" + return _because(decision.reason, notes[key]) if decision.outcome == ESCALATE: - if gate["honours_escalate"]: - return {"permission": words["escalate"]}, reason note = notes[_ESCALATE_FROM_TRANSFORM] if degraded_from(decision) == TRANSFORM else notes["escalate"] - return {"permission": words["block"]}, _because(reason, note % name) - return {"permission": words["allow"]}, reason + return _because(decision.reason, note % name) + return decision.reason + + +def _gate_payload(v, gate, decision, name): + """The PRE_TOOL gate's (permission, reason) pair, before the shared trailing message rule.""" + words = v["words"] + if decision.outcome == TRANSFORM and gate["honours_transform"] and decision.updated_input is not None: + return {"permission": words["allow"], "updated_input": decision.updated_input}, decision.reason + if decision.outcome == ESCALATE and gate["honours_escalate"]: + return {"permission": words["escalate"]}, decision.reason + if decision.outcome in (DENY, ESCALATE, TRANSFORM): + return {"permission": words["block"]}, _refusal_reason(v, gate, decision, name) + return {"permission": words["allow"]}, decision.reason def cursor_respond(cfg, decision, event): @@ -134,7 +143,7 @@ def cursor_respond(cfg, decision, event): if gate is None or canonical != PRE_TOOL: return "", 0 - payload, reason = _gate_payload(v["words"], v["degrade_notes"], gate, decision, name) + payload, reason = _gate_payload(v, gate, decision, name) if reason and payload["permission"] != v["words"]["allow"]: payload["user_message"] = reason payload["agent_message"] = reason diff --git a/src/agentseam/data/vendors/cursor.json b/src/agentseam/data/vendors/cursor.json index c37f9ea..8aec8aa 100644 --- a/src/agentseam/data/vendors/cursor.json +++ b/src/agentseam/data/vendors/cursor.json @@ -126,7 +126,8 @@ "degrade_notes": { "escalate": "%s cannot prompt for confirmation, so this is a block", "escalate_from_transform": "%s cannot modify a tool call, so this is a block", - "transform": "input requires modification, which this gate cannot express" + "transform": "input requires modification, which this gate cannot express", + "transform_missing_input": "no replacement input was supplied" }, "flag_note": "observed after the fact (%s cannot prevent it): %s", "flag_note_default": "policy violation", diff --git a/tests/fixtures/golden/cursor.json b/tests/fixtures/golden/cursor.json index da08832..9d6a433 100644 --- a/tests/fixtures/golden/cursor.json +++ b/tests/fixtures/golden/cursor.json @@ -145,7 +145,7 @@ }, "rewrite-without-input": { "exit": 0, - "stdout": "{\"permission\": \"deny\", \"user_message\": \"needs change (input requires modification, which this gate cannot express)\", \"agent_message\": \"needs change (input requires modification, which this gate cannot express)\"}" + "stdout": "{\"permission\": \"deny\", \"user_message\": \"needs change (no replacement input was supplied)\", \"agent_message\": \"needs change (no replacement input was supplied)\"}" }, "vouch": { "exit": 0, diff --git a/tests/test_adapter_cursor.py b/tests/test_adapter_cursor.py index 6a81d4e..9306179 100644 --- a/tests/test_adapter_cursor.py +++ b/tests/test_adapter_cursor.py @@ -140,3 +140,9 @@ def test_respond_infers_the_unnamed_event_the_same_way_parse_does(): mod = A.adapters.get("cursor") assert mod.parse(raw).event == A.FILE_CHANGED assert mod.respond(Decision.deny("no"), mod.parse(raw)) == ("", 0) + + +def test_a_rewrite_with_no_input_is_not_blamed_on_the_gate_that_can_rewrite(): + """preToolUse is the one gate that CAN express a rewrite; the handler supplied none.""" + text, _, _, _ = A.handle(CU_PRE_TOOL, lambda _e: Decision.rewrite(None, "needs change")) + assert json.loads(text)["user_message"] == "needs change (no replacement input was supplied)" diff --git a/tools/recount/tables.py b/tools/recount/tables.py index 127858b..6a2df8f 100644 --- a/tools/recount/tables.py +++ b/tools/recount/tables.py @@ -204,6 +204,7 @@ "escalate": "%s cannot prompt for confirmation, so this is a block", "escalate_from_transform": "%s cannot modify a tool call, so this is a block", "transform": "input requires modification, which this gate cannot express", + "transform_missing_input": "no replacement input was supplied", }, "flag_note": "observed after the fact (%s cannot prevent it): %s", "flag_note_default": "policy violation", From 9b550d5b061a562bc28bdbcd88e3e559f6dd68a3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:26:55 +0000 Subject: [PATCH 25/29] cursor: the prompt gate speaks a refusal, not an allow's rationale Finding raw[2].findings[6] of the vendor-truth review, both halves reproduced by execution. At beforeSubmitPrompt user_message was attached whenever the decision carried a reason, whatever the outcome, so a handler annotating its allows for its own audit trail put that text in Cursor's UI on every submitted prompt. The permission-gate branch has always attached messages only when not allowing. The same branch also skipped the degradation note every other gate adds, so a prompt blocked because a rewrite could not be expressed there was shown only the handler's original reason -- the mis-story _because() exists to prevent. It now shares _refusal_reason with the gate branch. The shared escalate_from_transform note says "cannot modify the input" rather than "cannot modify a tool call": a prompt gate modifies a prompt. No permission-gate scenario emitted that note, so the four frozen wire bytes that move are all at prompt_submit. Signed-off-by: Claude Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 13 +++++++++++++ src/agentseam/adapters/_cursor.py | 17 ++++++++++------- src/agentseam/data/vendors/cursor.json | 2 +- tests/fixtures/golden/cursor.json | 8 ++++---- tests/test_adapter_cursor.py | 14 ++++++++++++++ tools/recount/tables.py | 2 +- 6 files changed, 43 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ac3490..7b500bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,19 @@ versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html). re-run -- still show their original drift. ### Fixed +- **Cursor's prompt gate stops surfacing an allow's own rationale, and starts explaining a + refusal** (`adapters/_cursor.py`, `data/vendors/cursor.json`; vendor-truth review finding + `raw[2].findings[6]`). At `beforeSubmitPrompt`, `user_message` was attached whenever the + decision carried a reason, whatever the outcome -- so a handler annotating its allows for + its own audit trail (`Decision.allow("matched allowlist rule 7")`) put that text in + Cursor's UI on every submitted prompt. The permission-gate branch has always attached + messages only when not allowing; the prompt gate now does the same. It also runs the + refusal through the same degradation note the other gates use, so a prompt blocked + *because a rewrite could not be expressed there* says so instead of showing only the + handler's original reason. Four bytes of frozen wire output move, all at `prompt_submit`. + The shared `escalate_from_transform` note now says "cannot modify the input" rather than + "cannot modify a tool call", which is what a prompt gate is modifying; no permission-gate + scenario emitted that note, so nothing else moves. - **A rewrite with no replacement input is no longer blamed on Cursor's rewrite gate** (`adapters/_cursor.py`, `data/vendors/cursor.json`; vendor-truth review finding `raw[2].findings[4]`). `Decision.rewrite(None, ...)` at `preToolUse` was refused with diff --git a/src/agentseam/adapters/_cursor.py b/src/agentseam/adapters/_cursor.py index 6868af2..4977e85 100644 --- a/src/agentseam/adapters/_cursor.py +++ b/src/agentseam/adapters/_cursor.py @@ -94,10 +94,14 @@ def _flag_payload(v, decision, name): return _json.dumps({"additional_context": note}), 0 -def _prompt_submit_payload(decision): - payload = {"continue": decision.outcome not in (DENY, ESCALATE, TRANSFORM)} - if decision.reason: - payload["user_message"] = decision.reason +def _prompt_submit_payload(v, gate, decision, name): + """`user_message` is end-user text, so it carries a refusal -- never an allow's own + rationale, which the permission gate has never surfaced either.""" + blocking = decision.outcome in (DENY, ESCALATE, TRANSFORM) + payload = {"continue": not blocking} + reason = _refusal_reason(v, gate, decision, name) if blocking else None + if reason: + payload["user_message"] = reason return _json.dumps(payload), 0 @@ -136,10 +140,9 @@ def cursor_respond(cfg, decision, event): return "", 0 if canonical in (POST_TOOL, TOOL_FAILURE): return _flag_payload(v, decision, name) - if canonical == PROMPT_SUBMIT: - return _prompt_submit_payload(decision) - gate = v["gates"].get(name) + if canonical == PROMPT_SUBMIT and gate is not None: + return _prompt_submit_payload(v, gate, decision, name) if gate is None or canonical != PRE_TOOL: return "", 0 diff --git a/src/agentseam/data/vendors/cursor.json b/src/agentseam/data/vendors/cursor.json index 8aec8aa..46899b4 100644 --- a/src/agentseam/data/vendors/cursor.json +++ b/src/agentseam/data/vendors/cursor.json @@ -125,7 +125,7 @@ "default_wire_event": "beforeShellExecution", "degrade_notes": { "escalate": "%s cannot prompt for confirmation, so this is a block", - "escalate_from_transform": "%s cannot modify a tool call, so this is a block", + "escalate_from_transform": "%s cannot modify the input, so this is a block", "transform": "input requires modification, which this gate cannot express", "transform_missing_input": "no replacement input was supplied" }, diff --git a/tests/fixtures/golden/cursor.json b/tests/fixtures/golden/cursor.json index 9d6a433..283dd1f 100644 --- a/tests/fixtures/golden/cursor.json +++ b/tests/fixtures/golden/cursor.json @@ -177,7 +177,7 @@ }, "ask": { "exit": 0, - "stdout": "{\"continue\": false, \"user_message\": \"confirm\"}" + "stdout": "{\"continue\": false, \"user_message\": \"confirm (beforeSubmitPrompt cannot prompt for confirmation, so this is a block)\"}" }, "deny": { "exit": 0, @@ -185,15 +185,15 @@ }, "rewrite": { "exit": 0, - "stdout": "{\"continue\": false, \"user_message\": \"redacted\"}" + "stdout": "{\"continue\": false, \"user_message\": \"redacted (beforeSubmitPrompt cannot modify the input, so this is a block)\"}" }, "rewrite-without-input": { "exit": 0, - "stdout": "{\"continue\": false, \"user_message\": \"needs change\"}" + "stdout": "{\"continue\": false, \"user_message\": \"needs change (beforeSubmitPrompt cannot modify the input, so this is a block)\"}" }, "vouch": { "exit": 0, - "stdout": "{\"continue\": true, \"user_message\": \"trusted\"}" + "stdout": "{\"continue\": true}" } }, "payload": { diff --git a/tests/test_adapter_cursor.py b/tests/test_adapter_cursor.py index 9306179..abb92d6 100644 --- a/tests/test_adapter_cursor.py +++ b/tests/test_adapter_cursor.py @@ -146,3 +146,17 @@ def test_a_rewrite_with_no_input_is_not_blamed_on_the_gate_that_can_rewrite(): """preToolUse is the one gate that CAN express a rewrite; the handler supplied none.""" text, _, _, _ = A.handle(CU_PRE_TOOL, lambda _e: Decision.rewrite(None, "needs change")) assert json.loads(text)["user_message"] == "needs change (no replacement input was supplied)" + + +def test_an_allow_does_not_leak_its_own_rationale_to_the_prompt_gate(): + """user_message is end-user text. The permission gate has always attached it only when""" + text, _, _, _ = A.handle(CU_SUBMIT, lambda _e: Decision.allow("matched allowlist rule 7")) + assert json.loads(text) == {"continue": True} + + +def test_a_prompt_gate_refusal_says_why_the_outcome_changed_shape(): + """Every other cursor gate explains a degraded rewrite; this one told only the handler's""" + text, _, _, _ = A.handle(CU_SUBMIT, lambda _e: Decision.rewrite({"prompt": "clean"}, "sanitized")) + payload = json.loads(text) + assert payload["continue"] is False + assert payload["user_message"] == "sanitized (beforeSubmitPrompt cannot modify the input, so this is a block)" diff --git a/tools/recount/tables.py b/tools/recount/tables.py index 6a2df8f..37aef4e 100644 --- a/tools/recount/tables.py +++ b/tools/recount/tables.py @@ -202,7 +202,7 @@ "words": {"allow": "allow", "block": "deny", "escalate": "ask"}, "degrade_notes": { "escalate": "%s cannot prompt for confirmation, so this is a block", - "escalate_from_transform": "%s cannot modify a tool call, so this is a block", + "escalate_from_transform": "%s cannot modify the input, so this is a block", "transform": "input requires modification, which this gate cannot express", "transform_missing_input": "no replacement input was supplied", }, From aa59ed8866e1a9e975c5b3a1aeb7fd9e930ae543 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:29:26 +0000 Subject: [PATCH 26/29] adapters: escape control characters when rendering Kimi's TOML block Finding raw[8].findings[4] of the vendor-truth review, reproduced by execution. _toml_value escaped only backslash and double quote, so a command or matcher carrying a newline ended the TOML line and spilled the rest into the [[hooks]] table as extra bare keys -- against a vendor rule this repository records itself: four fields only, a fifth makes the whole file fail to load. The rendered block does not parse, and neither does a user's own [model] section above it, so every hook in the file stops firing on a vendor that fails open. Silent, and total. _toml_value now emits the full basic-string escape set with \uXXXX for every other C0 control and U+007F. The block is pinned by a round-trip through tomllib where it exists, and unconditionally by a line count derived from the rule's own field count -- the newline bug's signature. No output moves for a control-character-free command, which is every committed example and fixture. Signed-off-by: Claude Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 13 +++++++++++++ src/agentseam/adapters/_hook_entry.py | 14 ++++++++++++-- tests/test_adapter_kimi_code.py | 27 +++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebbb463..4fb9695 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,19 @@ versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html). re-run -- still show their original drift. ### Fixed +- **A control character in the installed command no longer makes Kimi Code's whole + `config.toml` unloadable** (`adapters/_hook_entry.py`; vendor-truth review finding + `raw[8].findings[4]`). `_toml_value` escaped only backslash and double quote, so a command + or matcher carrying a newline (a multi-line shell wrapper, an awkward path) ended the TOML + line and spilled the rest into the `[[hooks]]` table as extra bare keys -- against a vendor + rule this repository records itself: "four fields only; a fifth makes the whole file fail + to load". Reproduced by execution: the rendered block does not parse, and neither does a + user's own `[model]` section above it. The failure mode is total and silent -- every hook, + ours and the user's, stops firing on a vendor that fails open. `_toml_value` now emits the + full TOML basic-string escape set (`\b \t \n \f \r \" \\`) with `\uXXXX` for every + other C0 control and U+007F, and the rendered block is pinned by a round-trip through + `tomllib`. No output moves for any command that was already control-character free, which + is every committed example and fixture. - **The stop gate's block observable is the hook re-firing, not a second action run** (`tools/experiment.py`). `_blocked()` used to score a Stop-gate block by reading the sentinel twice, on the theory that an agent refused permission to finish comes back diff --git a/src/agentseam/adapters/_hook_entry.py b/src/agentseam/adapters/_hook_entry.py index c9fde1f..45bad9d 100644 --- a/src/agentseam/adapters/_hook_entry.py +++ b/src/agentseam/adapters/_hook_entry.py @@ -95,11 +95,21 @@ def hook_entry_config(cfg, canonical_events, command, matcher=None, *, fail_clos return _default_wrapper(cfg, reverse, canonical_events, command, matcher) +#: TOML basic-string escapes. Every other C0 control (and U+007F) takes the \\uXXXX form; +#: a raw one is a parse error, and Kimi rejects the WHOLE config.toml over a bad entry. +_TOML_ESCAPES = {"\\": "\\\\", '"': '\\"', "\b": "\\b", "\t": "\\t", "\n": "\\n", "\f": "\\f", "\r": "\\r"} + + +def _toml_char(ch): + if ch in _TOML_ESCAPES: + return _TOML_ESCAPES[ch] + return "\\u%04X" % ord(ch) if ch < " " or ch == "\x7f" else ch + + def _toml_value(value): if isinstance(value, int) and not isinstance(value, bool): return str(value) - escaped = str(value).replace("\\", "\\\\").replace('"', '\\"') - return '"%s"' % escaped + return '"%s"' % "".join(_toml_char(ch) for ch in str(value)) def render_config(rules): diff --git a/tests/test_adapter_kimi_code.py b/tests/test_adapter_kimi_code.py index bf7eb85..45877e9 100644 --- a/tests/test_adapter_kimi_code.py +++ b/tests/test_adapter_kimi_code.py @@ -4,6 +4,8 @@ import sys from pathlib import Path +import pytest + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from payloads import KM_NOTIFY, KM_POST, KM_SHELL, KM_WRITE # noqa: E402 @@ -131,3 +133,28 @@ def test_kimi_blocks_but_fails_open_and_the_notes_say_not_to_rely_on_it(): assert not A.can_rewrite("kimi_code", A.PRE_TOOL) assert A.enforcement_level("kimi_code", A.PRE_TOOL) == "best-effort" assert "not a sole security barrier" in A.MATRIX["kimi_code"]["notes"] + + +#: A command a shell wrapper or an awkward path could carry: quote, newline, CR, tab, and a +#: bare C0 control. The newline is the one that matters -- it ends the TOML line, so the rest +#: lands as extra bare keys in a table the vendor documents as taking four fields only. +_HOSTILE_COMMAND = 'py -c "x"\nevil = 1\ttab\rcr\x01ctl' + + +def test_a_control_character_in_the_command_still_renders_one_key_per_line(): + """Kimi rejects the WHOLE config.toml over one bad entry, so an unescaped newline does""" + mod = A.adapters.get("kimi_code") + rules = mod.hook_config([A.PRE_TOOL], _HOSTILE_COMMAND, matcher="Bash") + block = mod.render_config(rules).strip() + assert len(block.splitlines()) == 1 + len(rules[0]), block + assert not [ch for ch in block.replace("\n", "") if ch < " "], block + + +@pytest.mark.skipif(sys.version_info < (3, 11), reason="tomllib arrived in 3.11") +def test_the_rendered_block_parses_back_to_the_rules_it_was_given(): + """The derivation: a real TOML parser must return exactly what hook_config produced.""" + import tomllib + + mod = A.adapters.get("kimi_code") + rules = mod.hook_config([A.PRE_TOOL, A.STOP], _HOSTILE_COMMAND, matcher="Bash|Edit") + assert tomllib.loads(mod.render_config(rules))["hooks"] == list(rules) From a67ea55e3b0d72c095b070d011e0be96c22d7be6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:49:28 +0000 Subject: [PATCH 27/29] recorded driver: choose the default driver per gate, not per agent claude_code@2.1.263 records pre_tool only. resolve_driver() defaulted to 'recorded' whenever the agent had any recording, so a run at stop or prompt_submit died on NoRecording instead of falling back to the reference. has_recording() now takes the event (and version) and resolve_driver() passes them through; the CLI passes --event and --agent-version. Test covers the recorded gate, the two unrecorded gates, an agent with no recording, and an explicit --driver override. Also: the recording schema's top-level description was a JSON array, which the repo's own validator accepted but the 2020-12 meta-schema rejects; it is one string now. Found in review of #117. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Claude --- src/agentseam/data/recordings/schema.json | 84 ++++++++++++++++------- tests/test_recorded_driver.py | 10 +++ tools/experiment.py | 2 +- tools/recorded_driver.py | 20 ++++-- 4 files changed, 84 insertions(+), 32 deletions(-) diff --git a/src/agentseam/data/recordings/schema.json b/src/agentseam/data/recordings/schema.json index 30b7770..cdc48aa 100644 --- a/src/agentseam/data/recordings/schema.json +++ b/src/agentseam/data/recordings/schema.json @@ -2,28 +2,44 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/open-coder-ai/agentseam/blob/main/src/agentseam/data/recordings/schema.json", "title": "agentseam recorded driver: a frozen witness run", - "description": [ - "One file per (agent, version) -- data/recordings/@.json -- written by", - "tools/experiment.py --record from a real agent, never by hand and never edited after", - "landing: a newer version is a new file, not a rewrite of this one (owner decision", - "2026-09-07, org-plan plan/agentseam-project.md 'Evidence layer'). `tools/recorded_driver.py`", - "replays it through the same tools/experiment.py classifier a live run would have used,", - "so `--driver recorded` reproduces the witnessed table without launching a process.", - "Facts only: `notes` is the one line of prose this file allows." - ], + "description": "One file per (agent, version) -- data/recordings/@.json -- written by tools/experiment.py --record from a real agent, never by hand and never edited after landing: a newer version is a new file, not a rewrite of this one (owner decision 2026-09-07, org-plan plan/agentseam-project.md 'Evidence layer'). `tools/recorded_driver.py` replays it through the same tools/experiment.py classifier a live run would have used, so `--driver recorded` reproduces the witnessed table without launching a process. Facts only: `notes` is the one line of prose this file allows.", "type": "object", "additionalProperties": false, - "required": ["agent", "version", "platform", "date", "driver", "events"], + "required": [ + "agent", + "version", + "platform", + "date", + "driver", + "events" + ], "properties": { - "agent": { "type": "string", "pattern": "^[a-z][a-z0-9_]*$" }, - "version": { "type": "string", "minLength": 1 }, - "platform": { "type": "string", "minLength": 1 }, - "date": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}$" }, + "agent": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]*$" + }, + "version": { + "type": "string", + "minLength": 1 + }, + "platform": { + "type": "string", + "minLength": 1 + }, + "date": { + "type": "string", + "pattern": "^\\d{4}-\\d{2}-\\d{2}$" + }, "driver": { - "enum": ["real-agent"], + "enum": [ + "real-agent" + ], "description": "Always real-agent: the same invariant evidence_report.py enforces on a submitted report -- only a running agent may freeze a witness recording." }, - "reporter": { "type": "string", "minLength": 1 }, + "reporter": { + "type": "string", + "minLength": 1 + }, "notes": { "type": "string", "minLength": 1, @@ -33,40 +49,60 @@ "type": "object", "minProperties": 1, "description": "Canonical event name -> what was witnessed there. A key present here IS the claim 'this gate was measured'; a gate this agent claims but this file omits was simply not run yet -- that is the partial-recording case, not an error.", - "additionalProperties": { "$ref": "#/$defs/eventRecord" } + "additionalProperties": { + "$ref": "#/$defs/eventRecord" + } } }, "$defs": { "eventRecord": { "type": "object", "additionalProperties": false, - "required": ["trials", "payload_keys"], + "required": [ + "trials", + "payload_keys" + ], "properties": { "payload_keys": { "type": "array", - "items": { "type": "string", "minLength": 1 }, + "items": { + "type": "string", + "minLength": 1 + }, "description": "Every payload key seen across this event's trials, sorted and deduplicated -- the wire shape at this gate, not a per-trial value." }, "trials": { "type": "object", "minProperties": 1, "description": "Trial name (tools/experiment_probe.py's BEHAVIOURS, plus escalate when run) -> the sentinel and invocation counts witnessed. Not the classified outcome itself: replaying these through experiment._classify is what keeps a recording's semantics identical to a live run's, rather than merely similar.", - "additionalProperties": { "$ref": "#/$defs/trialRecord" } + "additionalProperties": { + "$ref": "#/$defs/trialRecord" + } } } }, "trialRecord": { "type": "object", "additionalProperties": false, - "required": ["observed", "hook_invocations"], + "required": [ + "observed", + "hook_invocations" + ], "properties": { "observed": { "type": "object", "additionalProperties": false, - "required": ["runs", "alt_runs"], + "required": [ + "runs", + "alt_runs" + ], "properties": { - "runs": { "type": "number" }, - "alt_runs": { "type": "number" } + "runs": { + "type": "number" + }, + "alt_runs": { + "type": "number" + } } }, "hook_invocations": { diff --git a/tests/test_recorded_driver.py b/tests/test_recorded_driver.py index e0ad240..7e86bda 100644 --- a/tests/test_recorded_driver.py +++ b/tests/test_recorded_driver.py @@ -174,3 +174,13 @@ def test_record_leaves_no_scratch_workspace_behind(): shutil.rmtree(results[0]["workspace"], ignore_errors=True) if path and os.path.exists(path): os.remove(path) + + +def test_the_default_driver_is_chosen_per_gate_not_per_agent(): + """claude_code@2.1.263 recorded pre_tool only: that gate replays, the others fall back to + the reference instead of failing on a recording that never saw them.""" + assert recorded_driver.resolve_driver("claude_code", None, event="pre_tool") == recorded_driver.DRIVER_NAME + assert recorded_driver.resolve_driver("claude_code", None, event="stop") == recorded_driver.REFERENCE_DRIVER + assert recorded_driver.resolve_driver("claude_code", None, event="prompt_submit") == recorded_driver.REFERENCE_DRIVER + assert recorded_driver.resolve_driver("tabnine", None, event="pre_tool") == recorded_driver.REFERENCE_DRIVER + assert recorded_driver.resolve_driver("claude_code", "reference", event="pre_tool") == recorded_driver.REFERENCE_DRIVER diff --git a/tools/experiment.py b/tools/experiment.py index bfb82a0..f6d16f0 100644 --- a/tools/experiment.py +++ b/tools/experiment.py @@ -269,7 +269,7 @@ def main(argv=None): print("%-10s %s" % (name, what)) return 0 - args.driver = recorded_driver.resolve_driver(args.agent, args.driver) + args.driver = recorded_driver.resolve_driver(args.agent, args.driver, event=args.event, version=args.agent_version) if args.record: recorded_driver.check_record_args(parser, driver=args.driver, agent_version=args.agent_version) diff --git a/tools/recorded_driver.py b/tools/recorded_driver.py index b328340..087be47 100644 --- a/tools/recorded_driver.py +++ b/tools/recorded_driver.py @@ -43,8 +43,13 @@ class NoRecording(Exception): """No recording covers the (agent, version, event, trial) asked for.""" -def has_recording(agent): - return recordings.latest_version(agent) is not None +def has_recording(agent, event=None, version=None): + """Whether a recording exists for `agent` (at `version`, else the latest) and, when `event` + is given, whether it covers that gate. A recording of one gate says nothing about another.""" + body = recordings.load_recording(agent, version) + if body is None: + return False + return event is None or event in (body.get("events") or {}) def add_cli_args(run_parser): @@ -53,19 +58,20 @@ def add_cli_args(run_parser): "--driver", default=None, help="'reference', 'recorded', or a shell template containing {prompt}; " - "default: 'recorded' if a recording exists for --agent, else 'reference'", + "default: 'recorded' if a recording covers --agent at --event, else 'reference'", ) run_parser.add_argument( "--record", action="store_true", help="freeze this run into data/recordings/@.json" ) -def resolve_driver(agent, driver): - """`driver` if given, else 'recorded' when a recording exists for `agent`, else the - reference. Centralised here so tools/experiment.py's CLI stays a thin dispatcher.""" +def resolve_driver(agent, driver, event=None, version=None): + """`driver` if given, else 'recorded' when a recording covers `agent` at `event`, else the + reference. Per gate, not per agent: claude_code@2.1.263 recorded pre_tool only, and a run + at stop must fall back to the reference rather than fail on a recording that never saw it.""" if driver is not None: return driver - return DRIVER_NAME if has_recording(agent) else REFERENCE_DRIVER + return DRIVER_NAME if has_recording(agent, event, version) else REFERENCE_DRIVER def check_record_args(parser, *, driver, agent_version): From dccc3c139076f21b71a58f0d41dacbb14092deb4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:49:53 +0000 Subject: [PATCH 28/29] style: ruff format the per-gate resolve_driver call Co-Authored-By: Claude Fable 5.1 Signed-off-by: Claude --- tests/test_recorded_driver.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_recorded_driver.py b/tests/test_recorded_driver.py index 7e86bda..87e81ee 100644 --- a/tests/test_recorded_driver.py +++ b/tests/test_recorded_driver.py @@ -181,6 +181,10 @@ def test_the_default_driver_is_chosen_per_gate_not_per_agent(): the reference instead of failing on a recording that never saw them.""" assert recorded_driver.resolve_driver("claude_code", None, event="pre_tool") == recorded_driver.DRIVER_NAME assert recorded_driver.resolve_driver("claude_code", None, event="stop") == recorded_driver.REFERENCE_DRIVER - assert recorded_driver.resolve_driver("claude_code", None, event="prompt_submit") == recorded_driver.REFERENCE_DRIVER + assert ( + recorded_driver.resolve_driver("claude_code", None, event="prompt_submit") == recorded_driver.REFERENCE_DRIVER + ) assert recorded_driver.resolve_driver("tabnine", None, event="pre_tool") == recorded_driver.REFERENCE_DRIVER - assert recorded_driver.resolve_driver("claude_code", "reference", event="pre_tool") == recorded_driver.REFERENCE_DRIVER + assert ( + recorded_driver.resolve_driver("claude_code", "reference", event="pre_tool") == recorded_driver.REFERENCE_DRIVER + ) From cdae8cca84211b2a3ee247c1f3113b53d5eb6486 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:55:00 +0000 Subject: [PATCH 29/29] integration: reconcile W55 and W57 on the shared files; CLI test reads the row's version from data Seven reviewed branches merged in review order (#114, #115, #116, #118, #119, #120, #117). CHANGELOG conflicts resolved by keeping every entry; the two code conflicts between W55 (report `event`) and W57 (recorded driver) resolved by keeping both optional fields and both checks, the pair moved into one _check_provenance() helper so validate() stays under the complexity ceiling. W57's `matrix --evidence` test hard-coded 2.1.247 for the row's version; #114 moved the row to 2.1.263, so the test now reads the version from the data. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Claude --- src/agentseam/evidence_report.py | 14 ++++++++++---- tests/test_adapter_kimi_code.py | 2 ++ tests/test_cli.py | 6 ++++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/agentseam/evidence_report.py b/src/agentseam/evidence_report.py index 6ed6c7e..fad86e1 100644 --- a/src/agentseam/evidence_report.py +++ b/src/agentseam/evidence_report.py @@ -118,10 +118,7 @@ def validate(report): "watched cannot be checked for drift later." % report["basis"] ) - if report.get("event") is not None and report["event"] not in EVENTS: - raise InvalidReportError("event %r is not one of: %s" % (report["event"], ", ".join(EVENTS))) - if report["driver"] == RECORDED_DRIVER: - _check_recorded_version(report) + _check_provenance(report) if not str(report["date"]).count("-") == 2: # noqa: PLR2004 raise InvalidReportError("date must be YYYY-MM-DD, got %r" % (report["date"],)) @@ -129,6 +126,15 @@ def validate(report): return report +def _check_provenance(report): + """The two optional provenance fields: a named gate must be canonical, and a recorded + driver's version may not outrun its recording (W55 and W57, reconciled at integration).""" + if report.get("event") is not None and report["event"] not in EVENTS: + raise InvalidReportError("event %r is not one of: %s" % (report["event"], ", ".join(EVENTS))) + if report["driver"] == RECORDED_DRIVER: + _check_recorded_version(report) + + def _check_recorded_version(report): """A replayed recording may only claim the version it actually replayed, or older.""" recorded_version = report.get("recorded_version") diff --git a/tests/test_adapter_kimi_code.py b/tests/test_adapter_kimi_code.py index 756b8e8..74d3cf7 100644 --- a/tests/test_adapter_kimi_code.py +++ b/tests/test_adapter_kimi_code.py @@ -163,6 +163,8 @@ def test_accept_any_name_rests_on_client_type_never_being_absent(): claims = A.adapters.get("kimi_code").CONFIG["claims"] assert claims["accept_any_name"] and None not in claims["client_types"] assert not A.adapters.get("kimi_code").claims({"hook_event_name": "TurnStarted"}) + + #: A command a shell wrapper or an awkward path could carry: quote, newline, CR, tab, and a #: bare C0 control. The newline is the one that matters -- it ends the TOML line, so the rest #: lands as extra bare keys in a table the vendor documents as taking four fields only. diff --git a/tests/test_cli.py b/tests/test_cli.py index a93e183..83ca3ff 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -42,8 +42,10 @@ def test_matrix_evidence_shows_the_recorded_version_beside_the_rows(): out = _run(["matrix", "--evidence"]) assert out.returncode == 0 line = next(row for row in out.stdout.splitlines() if row.startswith("claude_code ")) - assert "2.1.247" in line # the row's own verified.version - assert "2.1.263" in line # the newest committed recording + from agentseam import matrix, recordings + + assert matrix.MATRIX["claude_code"]["verified"]["version"] in line # the row's own verified.version + assert recordings.latest_version("claude_code") in line # the newest committed recording #: A portable `head -3`: read three lines, then exit and drop the read end of the pipe.