|
| 1 | +"""Cross-vendor conformance over the committed recordings: where a witnessed run lands. |
| 2 | +
|
| 3 | +Each new witnessed run has been worth something only in the abstract -- one more row moving |
| 4 | +off vendor-docs. This is the concrete thing it buys: with two agents recorded at the same |
| 5 | +gate, every trial becomes a comparison, and `conformance.classify` says whether a difference |
| 6 | +is a true vendor property or this layer leaking a dialect. |
| 7 | +
|
| 8 | +Deliberately honest about the current state rather than impressive. One agent is recorded, so |
| 9 | +every comparison here reports `undecidable` -- one vendor cannot differ from anything. That |
| 10 | +reads as an empty result and is the correct one; the moment a second recording lands the same |
| 11 | +command starts answering. Nothing is fabricated to fill the table in the meantime. |
| 12 | +
|
| 13 | +Replay, not launch: `recorded_driver.run_trial` runs each recorded trial back through the real |
| 14 | +classifier without starting a process, so this is free to run in CI and measures exactly what |
| 15 | +a live run measured. |
| 16 | +""" |
| 17 | + |
| 18 | +from __future__ import annotations |
| 19 | + |
| 20 | +from .. import conformance, recordings |
| 21 | +from . import recorded_driver |
| 22 | + |
| 23 | + |
| 24 | +def _measured(agent, trial, event): |
| 25 | + """The one value `agent`'s recording measured for `trial` at `event`, or None.""" |
| 26 | + try: |
| 27 | + result = recorded_driver.run_trial(agent, trial, event=event) |
| 28 | + except recorded_driver.NoRecording: |
| 29 | + return None |
| 30 | + measured = result.get("measured") or {} |
| 31 | + return next(iter(measured.values()), None) if measured else None |
| 32 | + |
| 33 | + |
| 34 | +def events_recorded(): |
| 35 | + """Every gate at least one recording covers, sorted.""" |
| 36 | + found = set() |
| 37 | + for agent in recordings.agents(): |
| 38 | + body = recordings.load_recording(agent) or {} |
| 39 | + found.update(body.get("events") or {}) |
| 40 | + return sorted(found) |
| 41 | + |
| 42 | + |
| 43 | +def trials_recorded(event): |
| 44 | + """Every trial at least one recording covers at `event`, sorted.""" |
| 45 | + found = set() |
| 46 | + for agent in recordings.agents(): |
| 47 | + body = recordings.load_recording(agent) or {} |
| 48 | + found.update(((body.get("events") or {}).get(event) or {}).get("trials") or {}) |
| 49 | + return sorted(found) |
| 50 | + |
| 51 | + |
| 52 | +def compare(event): |
| 53 | + """One row per trial at `event`: the per-agent verdicts and what their difference means.""" |
| 54 | + rows = [] |
| 55 | + for trial in trials_recorded(event): |
| 56 | + verdicts = {} |
| 57 | + for agent in recordings.agents(): |
| 58 | + value = _measured(agent, trial, event) |
| 59 | + if value is not None: |
| 60 | + verdicts[agent] = value |
| 61 | + if not verdicts: |
| 62 | + continue |
| 63 | + rows.append({"trial": trial, "verdicts": verdicts, "result": conformance.classify(verdicts, event)}) |
| 64 | + return rows |
| 65 | + |
| 66 | + |
| 67 | +def gaps(event=None): |
| 68 | + """Every comparison that condemns the seam. Empty is the answer a green CI gate wants.""" |
| 69 | + events = [event] if event else events_recorded() |
| 70 | + return [ |
| 71 | + {"event": ev, **row} for ev in events for row in compare(ev) if row["result"]["call"] == conformance.SEAM_GAP |
| 72 | + ] |
| 73 | + |
| 74 | + |
| 75 | +def render(event=None): |
| 76 | + """Print the comparison and return an exit code: non-zero only for a seam gap.""" |
| 77 | + witnessed = recordings.agents() |
| 78 | + print("recorded agents: %s" % (", ".join(witnessed) or "(none)")) |
| 79 | + if len(witnessed) < 2: # noqa: PLR2004 -- conformance._MIN_VENDORS, stated where it is read |
| 80 | + print( |
| 81 | + "a comparison needs two: with %d recorded, every trial below reads undecidable, which\n" |
| 82 | + "is the honest answer rather than an empty one. Witness a second agent at the same gate\n" |
| 83 | + "and these rows start deciding." % len(witnessed) |
| 84 | + ) |
| 85 | + events = [event] if event else events_recorded() |
| 86 | + found = 0 |
| 87 | + for ev in events: |
| 88 | + rows = compare(ev) |
| 89 | + if not rows: |
| 90 | + continue |
| 91 | + print("\n%s" % ev) |
| 92 | + for row in rows: |
| 93 | + verdicts = ", ".join("%s=%s" % (a, v) for a, v in sorted(row["verdicts"].items())) |
| 94 | + print(" %-10s %-12s %s" % (row["trial"], row["result"]["call"], verdicts)) |
| 95 | + if row["result"]["call"] == conformance.SEAM_GAP: |
| 96 | + found += 1 |
| 97 | + print(" %s" % row["result"]["reason"]) |
| 98 | + if found: |
| 99 | + print("\n%d seam gap(s): vendors the matrix says are equally able measured differently." % found) |
| 100 | + return 1 |
| 101 | + return 0 |
0 commit comments