Record the Cursor event gap (R3), and wire harness + conformance into the probe

docs/witness-skeleton.json

@@ -1,53 +1,53 @@
 {
-  "_note": "Generated by the armed initiative, wave 1 (W1, 2026-09-10). Each entry in `gaps` names one coverage gap the committed witness recording (src/agentseam/data/recordings/claude_code@2.1.263.json) does not close, the exact command that closes it, and the evidence-report shape (src/agentseam/evidence_report.py) that command's --report output takes. Copy one `report` object, run its `command`, and replace the blank fields with the real --report output verbatim -- do not hand-type a basis, date or version. tests/test_coverage_skeleton.py asserts every `report` below, exactly as committed, is REJECTED by evidence_report.validate() -- an unfilled block must never be mistaken for a witnessed one.",
-  "gaps": [
-    {
-      "gate": "pre_tool",
-      "trial": "escalate",
-      "why": "claude_code@2.1.263 covers every pre_tool trial except escalate -- the one PreToolUse reading (permissionDecision: \"ask\") no vendor doc settles, so the reference driver refuses to guess it (src/agentseam/probe/reference_agent.py raises Undocumented there).",
-      "command": "agentseam probe run --agent claude_code --event pre_tool --trial escalate --driver \"<your headless claude_code invocation, containing {prompt}>\" --agent-version <version> --record --report --reporter @yourhandle > escalate-pre_tool-report.json",
-      "report": {
-        "report_version": 1,
-        "agent": "claude_code",
-        "basis": "",
-        "date": "",
-        "driver": "real-agent",
-        "event": "pre_tool",
-        "version": "",
-        "experiments": {}
-      }
-    },
-    {
-      "gate": "prompt_submit",
-      "trial": null,
-      "why": "No trial has ever been witnessed at prompt_submit for any agent -- claude_code's block/fail_mode/etc. claims for this gate rest on vendor-docs only.",
-      "command": "agentseam probe run --agent claude_code --event prompt_submit --driver \"<your headless claude_code invocation, containing {prompt}>\" --agent-version <version> --record --report --reporter @yourhandle > prompt_submit-report.json",
-      "report": {
-        "report_version": 1,
-        "agent": "claude_code",
-        "basis": "",
-        "date": "",
-        "driver": "real-agent",
-        "event": "prompt_submit",
-        "version": "",
-        "experiments": {}
-      }
-    },
-    {
-      "gate": "stop",
-      "trial": null,
-      "why": "No trial has ever been witnessed at stop for any agent -- including the sentinel re-fire asymmetry (experiment.py's own _blocked()) that has so far only been exercised through the reference driver, never a real one.",
-      "command": "agentseam probe run --agent claude_code --event stop --driver \"<your headless claude_code invocation, containing {prompt}>\" --agent-version <version> --record --report --reporter @yourhandle > stop-report.json",
-      "report": {
-        "report_version": 1,
-        "agent": "claude_code",
-        "basis": "",
-        "date": "",
-        "driver": "real-agent",
-        "event": "stop",
-        "version": "",
-        "experiments": {}
-      }
-    }
-  ]
+ "_note": "Generated by the armed initiative, wave 1 (W1, 2026-09-10). Each entry in `gaps` names one coverage gap the committed witness recording (src/agentseam/data/recordings/claude_code@2.1.263.json) does not close, the exact command that closes it, and the evidence-report shape (src/agentseam/evidence_report.py) that command's --report output takes. Copy one `report` object, run its `command`, and replace the blank fields with the real --report output verbatim. `--driver harness` reads the invocation agentseam.harness records for the agent, so there is no driver string to write by hand; pass an explicit --driver instead if your install needs different flags -- do not hand-type a basis, date or version. tests/test_coverage_skeleton.py asserts every `report` below, exactly as committed, is REJECTED by evidence_report.validate() -- an unfilled block must never be mistaken for a witnessed one.",
+ "gaps": [
+  {
+   "gate": "pre_tool",
+   "trial": "escalate",
+   "why": "claude_code@2.1.263 covers every pre_tool trial except escalate -- the one PreToolUse reading (permissionDecision: \"ask\") no vendor doc settles, so the reference driver refuses to guess it (src/agentseam/probe/reference_agent.py raises Undocumented there).",
+   "command": "agentseam probe run --agent claude_code --event pre_tool --trial escalate --driver harness --agent-version <version> --record --report --reporter @yourhandle > escalate-pre_tool-report.json",
+   "report": {
+    "report_version": 1,
+    "agent": "claude_code",
+    "basis": "",
+    "date": "",
+    "driver": "real-agent",
+    "event": "pre_tool",
+    "version": "",
+    "experiments": {}
+   }
+  },
+  {
+   "gate": "prompt_submit",
+   "trial": null,
+   "why": "No trial has ever been witnessed at prompt_submit for any agent -- claude_code's block/fail_mode/etc. claims for this gate rest on vendor-docs only.",
+   "command": "agentseam probe run --agent claude_code --event prompt_submit --driver harness --agent-version <version> --record --report --reporter @yourhandle > prompt_submit-report.json",
+   "report": {
+    "report_version": 1,
+    "agent": "claude_code",
+    "basis": "",
+    "date": "",
+    "driver": "real-agent",
+    "event": "prompt_submit",
+    "version": "",
+    "experiments": {}
+   }
+  },
+  {
+   "gate": "stop",
+   "trial": null,
+   "why": "No trial has ever been witnessed at stop for any agent -- including the sentinel re-fire asymmetry (experiment.py's own _blocked()) that has so far only been exercised through the reference driver, never a real one.",
+   "command": "agentseam probe run --agent claude_code --event stop --driver harness --agent-version <version> --record --report --reporter @yourhandle > stop-report.json",
+   "report": {
+    "report_version": 1,
+    "agent": "claude_code",
+    "basis": "",
+    "date": "",
+    "driver": "real-agent",
+    "event": "stop",
+    "version": "",
+    "experiments": {}
+   }
+  }
+ ]
 }

pyproject.toml

@@ -132,6 +132,7 @@ exclude = ["*.py.tmpl"]
 "src/agentseam/probe/experiment.py" = ["PLR0913"]
 "src/agentseam/probe/experiment_cli.py" = ["T201"]
 "src/agentseam/probe/experiment_report.py" = ["T201"]
+"src/agentseam/probe/conformance_report.py" = ["T201"]
 "src/agentseam/probe/recorded_driver.py" = ["T201", "PLR0913", "N818"]
 "src/agentseam/probe/reference_agent.py" = ["PLR0913", "N818"]
 # TODO(lint-adoption): this wave's Sonar/Checkstyle/FindBugs baseline (coding-standards.md

src/agentseam/adapters/_hook_json.py

@@ -22,7 +22,19 @@
 
 
 def hj_reverse(cfg):
-    """Canonical event -> wire name: the naive inverse, then the entry's pinned overrides."""
+    """Canonical event -> wire name: the naive inverse, then the entry's pinned overrides.
+
+    One wire name per canonical event, which is a real limit and not an oversight (R3, gap 4).
+    Cursor is where it bites: `pre_tool` pins to `preToolUse`, and Cursor honours `ask` only at
+    `beforeShellExecution` / `beforeMCPExecution`. So installing at `pre_tool` forecloses `ask`
+    before dispatch is ever reached -- the runtime degrade to `deny` is honest about it, but the
+    install already chose. Deny-style policies, which is all that ships today, are unaffected.
+
+    The day an ask-style Cursor policy exists, this map has to select by decision dialect
+    (deny -> `preToolUse`, ask -> `beforeShellExecution`) and become one-to-many. Recorded here
+    rather than built, so the resolution is not re-litigated from scratch; the three facts it
+    rests on are pinned in tests/test_cursor_ask_dialect.py.
+    """
     reverse = {}
     for name, canonical in cfg["events"].items():
         if canonical != UNKNOWN:

src/agentseam/conformance.py

@@ -52,6 +52,9 @@
 #: Raised as a message constant so the text is not a literal inside the raise (TRY003).
 _NOTHING_TO_COMPARE = "no verdicts to compare: a policy that ran nowhere has no conformance"
 
+#: Below this, there is no cross-vendor question to answer.
+_MIN_VENDORS = 2
+
 
 class UnrecordedVendorError(LookupError):
     """A vendor in the comparison has no matrix row, so its divergence cannot be judged."""
@@ -85,6 +88,21 @@ def classify(verdicts, event, *, needs=NEEDS_BLOCK):
         raise ValueError(_NOTHING_TO_COMPARE)
     groups = _groups(verdicts)
 
+    # One vendor is not a comparison. It will always "agree", and reporting that as AGREED
+    # claims a cross-vendor check that never happened -- the same mistake as reading unanimity
+    # among incapable vendors as agreement, one row further out. Note this is about the number
+    # of vendors ASKED, not how many can enforce: one capable vendor beside one excused one is
+    # a real vendor-limit finding and still classifies as one.
+    if len(verdicts) < _MIN_VENDORS:
+        return _result(
+            UNDECIDABLE,
+            groups,
+            event,
+            (),
+            reason="only %s was asked: conformance is a comparison, and one vendor cannot differ "
+            "from anything" % next(iter(verdicts)),
+        )
+
     try:
         excused = tuple(sorted(a for a in verdicts if not capable(a, event, needs=needs)))
     except UnrecordedVendorError as exc:

src/agentseam/harness.py

@@ -23,6 +23,10 @@
 
 from __future__ import annotations
 
+import os as _os
+import shlex as _shlex
+import subprocess as _subprocess
+
 from ._data import load
 
 _DATA = load("harness.json")
@@ -39,6 +43,10 @@
 #: makes a run a measurement of the instructions rather than of the policy.
 INSTRUCTION_MARKERS = tuple(_DATA["instruction_markers"])
 
+#: The token the probe's driver template carries where the prompt goes. Named by the probe,
+#: not by us -- experiment_driver.drive_real() substitutes a JSON-quoted string for it.
+DRIVER_PROMPT_SLOT = "{prompt}"
+
 PROMPT_PLACEHOLDER = "__PROMPT__"
 MODEL_PLACEHOLDER = "__MODEL__"
 
@@ -128,3 +136,32 @@ def isolation(agent):
 def creates_files_as(agent):
     """Recorded prose on file ownership of what the agent creates, or None."""
     return row(agent).get("creates_files_as")
+
+
+def driver_command(agent, *, model=None, hooked=False):
+    """The recorded invocation as a probe driver template, with `{prompt}` where the prompt goes.
+
+    This is what removes the hand-typed driver string from a witness run: the registry already
+    knows how to reach the vendor, so the operator names the agent and nothing else.
+
+    Quoted for the platform actually running it. `shlex.quote` is POSIX-only, and these runs
+    happen on a Windows machine as often as not -- a POSIX-quoted argument there is not a
+    slightly-wrong command line, it is a different one. The prompt slot is left unquoted on
+    purpose: `drive_real` substitutes an already-quoted JSON string for it.
+    """
+    slot = "\x00agentseam-prompt\x00"
+    parts = argv(agent, slot, model=model, hooked=hooked)
+    rendered = [DRIVER_PROMPT_SLOT if part == slot else _quote_one(part) for part in parts]
+    if any(slot in part for part in rendered):
+        raise NoHarnessError(
+            "%r builds its prompt inside a larger argument, which this template cannot "
+            "express; drive it with an explicit --driver string." % agent
+        )
+    return " ".join(rendered)
+
+
+def _quote_one(part):
+    """Shell-quote one argument the way the running platform's shell reads it."""
+    if _os.name == "nt":
+        return _subprocess.list2cmdline([part])
+    return _shlex.quote(part)

src/agentseam/probe/cli.py

@@ -9,7 +9,7 @@
 
 import json
 
-from . import experiment, experiment_probe, experiment_report, recorded_driver
+from . import conformance_report, experiment, experiment_probe, experiment_report, recorded_driver
 
 
 def cmd_list(_args):
@@ -48,6 +48,11 @@ def cmd_run(args):
     return experiment_report.render(results, agent=args.agent, event=args.event, driver=driver)
 
 
+def cmd_conformance(args):
+    """`agentseam probe conformance`: compare the recorded agents, gate on seam gaps."""
+    return conformance_report.render(args.event)
+
+
 def add_subparser(sub, *, default_event):
     """Wire `probe list`/`probe run` onto `sub` -- agentseam.cli's own top-level subparsers."""
     pr = sub.add_parser("probe", help="measure what an agent's hooks actually enforce, against a driver")
@@ -70,3 +75,7 @@ def add_subparser(sub, *, default_event):
     run.add_argument("--agent-version", help="the agent build these trials ran against")
     run.add_argument("--reporter", help="how you want crediting, e.g. @handle")
     run.set_defaults(fn=cmd_run, _parser=run)
+
+    conf = psub.add_parser("conformance", help="compare the recorded agents; exit non-zero on a seam gap")
+    conf.add_argument("--event", default=None, choices=experiment.EVENTS, help="one gate (default: all recorded)")
+    conf.set_defaults(fn=cmd_conformance)

src/agentseam/probe/conformance_report.py

@@ -0,0 +1,101 @@
+"""Cross-vendor conformance over the committed recordings: where a witnessed run lands.
+
+Each new witnessed run has been worth something only in the abstract -- one more row moving
+off vendor-docs. This is the concrete thing it buys: with two agents recorded at the same
+gate, every trial becomes a comparison, and `conformance.classify` says whether a difference
+is a true vendor property or this layer leaking a dialect.
+
+Deliberately honest about the current state rather than impressive. One agent is recorded, so
+every comparison here reports `undecidable` -- one vendor cannot differ from anything. That
+reads as an empty result and is the correct one; the moment a second recording lands the same
+command starts answering. Nothing is fabricated to fill the table in the meantime.
+
+Replay, not launch: `recorded_driver.run_trial` runs each recorded trial back through the real
+classifier without starting a process, so this is free to run in CI and measures exactly what
+a live run measured.
+"""
+
+from __future__ import annotations
+
+from .. import conformance, recordings
+from . import recorded_driver
+
+
+def _measured(agent, trial, event):
+    """The one value `agent`'s recording measured for `trial` at `event`, or None."""
+    try:
+        result = recorded_driver.run_trial(agent, trial, event=event)
+    except recorded_driver.NoRecording:
+        return None
+    measured = result.get("measured") or {}
+    return next(iter(measured.values()), None) if measured else None
+
+
+def events_recorded():
+    """Every gate at least one recording covers, sorted."""
+    found = set()
+    for agent in recordings.agents():
+        body = recordings.load_recording(agent) or {}
+        found.update(body.get("events") or {})
+    return sorted(found)
+
+
+def trials_recorded(event):
+    """Every trial at least one recording covers at `event`, sorted."""
+    found = set()
+    for agent in recordings.agents():
+        body = recordings.load_recording(agent) or {}
+        found.update(((body.get("events") or {}).get(event) or {}).get("trials") or {})
+    return sorted(found)
+
+
+def compare(event):
+    """One row per trial at `event`: the per-agent verdicts and what their difference means."""
+    rows = []
+    for trial in trials_recorded(event):
+        verdicts = {}
+        for agent in recordings.agents():
+            value = _measured(agent, trial, event)
+            if value is not None:
+                verdicts[agent] = value
+        if not verdicts:
+            continue
+        rows.append({"trial": trial, "verdicts": verdicts, "result": conformance.classify(verdicts, event)})
+    return rows
+
+
+def gaps(event=None):
+    """Every comparison that condemns the seam. Empty is the answer a green CI gate wants."""
+    events = [event] if event else events_recorded()
+    return [
+        {"event": ev, **row} for ev in events for row in compare(ev) if row["result"]["call"] == conformance.SEAM_GAP
+    ]
+
+
+def render(event=None):
+    """Print the comparison and return an exit code: non-zero only for a seam gap."""
+    witnessed = recordings.agents()
+    print("recorded agents: %s" % (", ".join(witnessed) or "(none)"))
+    if len(witnessed) < 2:  # noqa: PLR2004 -- conformance._MIN_VENDORS, stated where it is read
+        print(
+            "a comparison needs two: with %d recorded, every trial below reads undecidable, which\n"
+            "is the honest answer rather than an empty one. Witness a second agent at the same gate\n"
+            "and these rows start deciding." % len(witnessed)
+        )
+    events = [event] if event else events_recorded()
+    found = 0
+    for ev in events:
+        rows = compare(ev)
+        if not rows:
+            continue
+        print("\n%s" % ev)
+        for row in rows:
+            verdicts = ", ".join("%s=%s" % (a, v) for a, v in sorted(row["verdicts"].items()))
+            print("  %-10s %-12s %s" % (row["trial"], row["result"]["call"], verdicts))
+            if row["result"]["call"] == conformance.SEAM_GAP:
+                found += 1
+                print("             %s" % row["result"]["reason"])
+    if found:
+        print("\n%d seam gap(s): vendors the matrix says are equally able measured differently." % found)
+        return 1
+    return 0

src/agentseam/probe/recorded_driver.py

@@ -25,7 +25,7 @@
 import sys
 from datetime import date
 
-from .. import recordings
+from .. import harness, recordings
 from ..evidence_report import RECORDED_DRIVER, REFERENCE_DRIVER
 
 #: The driver name tools/experiment.py dispatches on.
@@ -35,6 +35,11 @@
 #: evidence -- the same rule evidence_report.py enforces for a submitted report.
 NON_LIVE_DRIVERS = (REFERENCE_DRIVER, DRIVER_NAME)
 
+#: Asks for the invocation `agentseam.harness` records for this agent, instead of a template the
+#: operator types. Opt-in by name rather than a new default: the resolution order below is what
+#: decides whether a run is evidence, and it should not change because a registry gained a row.
+HARNESS_DRIVER = "harness"
+
 
 class NoRecording(Exception):
     """No recording covers the (agent, version, event, trial) asked for."""
@@ -54,8 +59,9 @@ def add_cli_args(run_parser):
     run_parser.add_argument(
         "--driver",
         default=None,
-        help="'reference', 'recorded', or a shell template containing {prompt}; "
-        "default: 'recorded' if a recording covers --agent at --event, else 'reference'",
+        help="'reference', 'recorded', 'harness' (the invocation agentseam.harness records "
+        "for --agent), or a shell template containing {prompt}; default: 'recorded' if a "
+        "recording covers --agent at --event, else 'reference'",
     )
     run_parser.add_argument(
         "--record", action="store_true", help="freeze this run into data/recordings/<agent>@<agent-version>.json"
@@ -66,6 +72,12 @@ def resolve_driver(agent, driver, event=None, version=None):
     """`driver` if given, else 'recorded' when a recording covers `agent` at `event`, else the
     reference. Per gate, not per agent: claude_code@2.1.263 recorded pre_tool only, and a run
     at stop must fall back to the reference rather than fail on a recording that never saw it."""
+    if driver == HARNESS_DRIVER:
+        # Expanded here, not carried as a token: everything downstream -- check_record_args,
+        # the rendered table, the report's `driver` field -- then sees a real command line and
+        # treats the run as live, which it is. A token would have had to be special-cased in
+        # each of those places to avoid being mistaken for a non-live driver.
+        return harness.driver_command(agent)
     if driver is not None:
         return driver
     return DRIVER_NAME if has_recording(agent, event, version) else REFERENCE_DRIVER