Stop the matrix reporting verdicts on guards it could not measure

Two defects, both of which let a run state a result it had not established.

Outcomes were inferred rather than read. Only the `FAILED` lines were parsed
and every other named test was recorded as passed, so a skipped test, a test
whose fixture raised, a deselected test and a test pytest never mentioned were
all indistinguishable from a passing one — the confusion `failing_tests`
already refuses for a whole suite, left open for a single test. With `-rA` each
test states an outcome; anything that is not plainly passed or failed is now
unclear, an unclear baseline records `no baseline` rather than proceeding to
measure against a precondition never met, and an unclear guard-off run records
`no verdict` rather than `does not discriminate`.

The test's name was taken from the wrong end of the line. A summary line is
`FAILED <nodeid> - <message>`, the message in these suites is a slice of
Warpgate's own log, and that log carries Rust module paths — so splitting on
the last `::` returned `logging:` and `config:` instead of a test name, and the
test was not recognised as having failed. Two guards were reported as failing
to discriminate on exactly this; both do discriminate, in CI and locally.

Also: the gateway binary is fingerprinted across the guard-off build, because
an A/B whose halves ran the same binary is not an A/B and reads as a coverage
hole.

Verified: 53 of 53 guards discriminate, every guard measured, shards accounting
for the whole table — https://github.com/janisdombr/warpgate/actions/runs/32802468772

Author: janisdombr

tests/mutation_matrix.py

@@ -46,6 +46,7 @@
 
 import ast
 import atexit
+import hashlib
 import json
 import pathlib
 import signal
@@ -791,8 +792,54 @@ def run(command, **kwargs):
     return subprocess.run(command, cwd=REPO, capture_output=True, text=True, **kwargs)
 
 
-def run_named_only(tests: list[str], crates=None) -> tuple[set[str], set[str]]:
-    """Run only the given tests. Returns (passed, failed), by name.
+def _test_name_from_summary_line(line: str, head: str) -> str | None:
+    """The test named by one of pytest's short-summary lines.
+
+    Read `line.split("::")[-1].split()[0]` and it looks right until a test
+    fails on an assertion whose message contains a Rust module path. A summary
+    line is `FAILED <nodeid> - <message>`, the message here is a slice of
+    Warpgate's own log, and the log is full of `warpgate_common_http::logging`.
+    Splitting on the *last* `::` then returns a fragment of that message, the
+    test is not recognised as having failed, and the run records it as passed —
+    which is how four guards that discriminate were reported as guards that do
+    not. The nodeid is what to read, and it ends at the first ` - `.
+    """
+    nodeid = line[len(head):].strip().split(" - ", 1)[0].strip()
+    if not nodeid:
+        return None
+    # `test_x[a b]` is one nodeid with a space in it; the parameters are not
+    # part of the name a guard names.
+    return nodeid.split("::")[-1].split("[", 1)[0].strip() or None
+
+
+def _gateway_fingerprint() -> str | None:
+    """Whether the binary the tests will run is the one just built.
+
+    An A/B where both halves ran the same binary is not an A/B, and it reports
+    `does not discriminate` — the guard\'s own test passing with the guard off —
+    which is indistinguishable from a real coverage hole. Two guards were
+    reported that way by the first CI sweep while both discriminated locally,
+    and nothing in the run recorded enough to tell the two explanations apart.
+    """
+    binary = REPO / "target" / "debug" / "warpgate"
+    if not binary.exists():
+        return None
+    digest = hashlib.sha256()
+    with binary.open("rb") as f:
+        for block in iter(lambda: f.read(1 << 20), b""):
+            digest.update(block)
+    return digest.hexdigest()[:16]
+
+
+def run_named_only(
+    tests: list[str], crates=None
+) -> tuple[set[str], set[str], dict[str, str]]:
+    """Run only the given tests. Returns (passed, failed, unclear), by name.
+
+    `unclear` is the third answer this used to lack. A test can be skipped,
+    error in a fixture, be deselected, or never be collected at all, and none
+    of those is a pass — but all of them were recorded as one, because the
+    only thing read was the list of failures.
 
     The whole suite is not run. That is the point: `run_one` runs everything
     because it asks "which tests notice", and the answer to that question is
@@ -810,6 +857,7 @@ def run_named_only(tests: list[str], crates=None) -> tuple[set[str], set[str]]:
     """
     passed: set[str] = set()
     failed: set[str] = set()
+    unclear: dict[str, str] = {}
 
     crates = crates or list(RUST_CRATES)
     rust = [t for t in tests if not t.startswith("test_") or _is_rust(t, crates)]
@@ -833,21 +881,39 @@ def run_named_only(tests: list[str], crates=None) -> tuple[set[str], set[str]]:
 
     if python:
         result = subprocess.run(
-            ["poetry", "run", "pytest", *SUITES, "-q", "--tb=no", "-p", "no:randomly",
-             "-k", " or ".join(python)],
+            ["poetry", "run", "pytest", *SUITES, "-q", "--tb=no", "-rA",
+             "-p", "no:randomly", "-k", " or ".join(python)],
             cwd=REPO / "tests",
             capture_output=True,
             text=True,
         )
-        reported = {
-            line.split("::")[-1].split()[0]
-            for line in result.stdout.splitlines()
-            if line.startswith("FAILED")
-        }
+        # Classify explicitly. Reading only the `FAILED` lines and calling
+        # everything else passed made a skipped, errored, deselected or
+        # never-collected test indistinguishable from a passing one — the same
+        # confusion `failing_tests` already refuses for a whole suite, left open
+        # for a single test. `-rA` makes pytest state an outcome per test, and a
+        # name pytest never mentions is recorded as unknown rather than passed.
+        outcome = {}
+        for line in result.stdout.splitlines():
+            head = line.split(" ", 1)[0]
+            if head in ("PASSED", "FAILED", "ERROR", "SKIPPED", "XFAIL", "XPASS"):
+                name = _test_name_from_summary_line(line, head)
+                if name:
+                    outcome[name] = head
         for name in python:
-            (failed if name in reported else passed).add(name)
+            verdict = outcome.get(name)
+            if verdict == "FAILED":
+                failed.add(name)
+            elif verdict == "PASSED":
+                passed.add(name)
+            else:
+                unclear[name] = verdict or "never reported by pytest"
+        if unclear:
+            # Kept because the reason a test did not report is in pytest's own
+            # output, and every run so far has thrown that output away.
+            unclear["pytest output"] = result.stdout[-1200:]
 
-    return passed, failed
+    return passed, failed, unclear
 
 
 _RUST_TEST_CACHE: dict[str, dict[str, str]] = {}
@@ -985,14 +1051,26 @@ def record(result):
             binary_is_mutated = False
 
         print(f"[{index:>2}/{total}]   baseline", flush=True)
-        before_pass, before_fail = run_named_only(expected, nearest)
+        before_pass, before_fail, before_unclear = run_named_only(expected, nearest)
         if before_fail:
             record({
                 "guard": name,
                 "status": "already failing",
                 "tests": sorted(before_fail),
             })
             continue
+        # The precondition is that the named test *passes* before the mutation.
+        # A test that was skipped, errored or never collected has not met it,
+        # and measuring the other half of the A/B against it produces a verdict
+        # about nothing. This used to read as a clean baseline.
+        if before_unclear:
+            record({
+                "guard": name,
+                "status": "no baseline",
+                "expected": expected,
+                "unclear_at_baseline": before_unclear,
+            })
+            continue
 
         source = REPO / path
         original = source.read_text()
@@ -1001,26 +1079,46 @@ def record(result):
         try:
             if needs_binary:
                 print(f"[{index:>2}/{total}]   building the gateway with the guard off", flush=True)
+                before_build = _gateway_fingerprint()
                 build = run(["cargo", "build", "--bin", "warpgate"])
                 binary_is_mutated = True
                 if build.returncode != 0:
                     record({"guard": name, "status": "did not compile"})
                     continue
+                after_build = _gateway_fingerprint()
+                if after_build is not None and after_build == before_build:
+                    record({
+                        "guard": name,
+                        "status": "the mutation never reached the binary",
+                        "expected": expected,
+                        "fingerprint": after_build,
+                        "build_output": build.stdout[-600:] + build.stderr[-600:],
+                    })
+                    continue
             print(f"[{index:>2}/{total}]   testing with the guard off", flush=True)
-            after_pass, after_fail = run_named_only(expected, nearest)
+            after_pass, after_fail, after_unclear = run_named_only(expected, nearest)
             # `all`, per A2: every test the entry names has to notice.
             status = (
                 "discriminates"
                 if set(expected) <= after_fail
+                else "no verdict"
+                if after_unclear
                 else "does not discriminate"
             )
-            record({
+            entry = {
                 "guard": name,
                 "status": status,
                 "expected": expected,
                 "failed_with_guard_off": sorted(after_fail),
                 "passed_with_guard_off": sorted(after_pass),
-            })
+            }
+            # Only for the verdicts that need explaining. A guard that
+            # discriminated needs no forensics; one that did not is exactly
+            # where the run has been unable to say why.
+            if status != "discriminates":
+                entry["unclear_with_guard_off"] = after_unclear
+                entry["gateway_built_for_this_guard"] = bool(needs_binary)
+            record(entry)
         finally:
             source.write_text(original)
             IN_FLIGHT.pop(str(source), None)
@@ -1059,9 +1157,13 @@ def failing_tests() -> tuple[set[str], str]:
     if "no tests ran" in result.stdout or "collected 0 items" in result.stdout:
         raise SystemExit(f"the suite collected nothing, so nothing was measured:\n{result.stdout[-500:]}")
     failed = {
-        line.split("::")[-1].split()[0]
+        name
         for line in result.stdout.splitlines()
         if line.startswith("FAILED")
+        # Same reason as in `run_named_only`: the last `::` on a summary line
+        # belongs to whatever Rust module path the assertion message quoted.
+        for name in [_test_name_from_summary_line(line, "FAILED")]
+        if name
     }
     return failed, result.stdout[-400:]