|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""P-037 B1: the first guarded shadow run over the frozen population. |
| 3 | +
|
| 4 | +Contract: docs/notes/p037-phase-b1-shadow.md (§B N5/N6, §E). Every input is |
| 5 | +read from git at the frozen population commit and nothing else (G14). Per |
| 6 | +document the extractor runs once with the opt-in A14 side report, and |
| 7 | +`own-guarded-report` reads the facts on stdin (R-1). The run aggregates: |
| 8 | +
|
| 9 | +* the summary and application class counts and the NO_GUARDED_EVIDENCE census; |
| 10 | +* every UNEXPLAINED row, tagged with whether its legacy value comes from the |
| 11 | + extractor's `ConsumesParam` fold (information only, never a class); |
| 12 | +* the A14 exposure metrics (side report joined to the shadow rows); |
| 13 | +* the R record-absence metrics. |
| 14 | +
|
| 15 | +Run: python scripts/p037_b1_shadow.py --out docs/evidence/p037-b1-shadow.json |
| 16 | + (needs dotnet, a built extractor and `cargo build --release -p own-shadow`) |
| 17 | +""" |
| 18 | +from __future__ import annotations |
| 19 | + |
| 20 | +import argparse |
| 21 | +import collections |
| 22 | +import hashlib |
| 23 | +import json |
| 24 | +import subprocess |
| 25 | +import sys |
| 26 | +import tempfile |
| 27 | +from pathlib import Path |
| 28 | +from typing import Any |
| 29 | + |
| 30 | +ROOT = Path(__file__).resolve().parent.parent |
| 31 | +sys.path.insert(0, str(ROOT / "scripts")) |
| 32 | + |
| 33 | +import p037_evidence as ev # noqa: E402 |
| 34 | + |
| 35 | +POPULATION_COMMIT = "571669e" |
| 36 | +EXTRACTOR = ROOT / "frontend/roslyn/OwnSharp.Extractor/bin/Debug/net8.0/ownsharp-extract.dll" |
| 37 | +REPORT_BIN = ROOT / "rust/target/release/own-guarded-report" |
| 38 | +CORPUS = ("corpus/real-world", "corpus/wpf", "corpus/di", "corpus/fixtures", "corpus/p036-bakeoff") |
| 39 | +CONTROLS = ("guarded-consume-", "gv4-control-", "legacy-honesty-") |
| 40 | +SIDE_KEYS = {"caller", "site", "callee", "dispatch", "observed_targets"} |
| 41 | + |
| 42 | + |
| 43 | +class Refused(RuntimeError): |
| 44 | + """The run cannot be a measurement of the frozen population.""" |
| 45 | + |
| 46 | + |
| 47 | +def git(*args: str) -> str: |
| 48 | + return subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, |
| 49 | + check=True).stdout |
| 50 | + |
| 51 | + |
| 52 | +def frozen(commit: str) -> str: |
| 53 | + """G14: only the frozen population commit is a measurement.""" |
| 54 | + want = git("rev-parse", f"{POPULATION_COMMIT}^{{commit}}").strip() |
| 55 | + got = git("rev-parse", f"{commit}^{{commit}}").strip() |
| 56 | + if got != want: |
| 57 | + raise Refused(f"population commit {commit} is not the frozen {POPULATION_COMMIT}; " |
| 58 | + "population drift is not a measurement (G14)") |
| 59 | + return want |
| 60 | + |
| 61 | + |
| 62 | +def validate_side_report(doc: Any) -> list[str]: |
| 63 | + """G15: the side report is measurement only, never a conclusion.""" |
| 64 | + if not isinstance(doc, dict): |
| 65 | + return ["side report is not an object"] |
| 66 | + bad = [] |
| 67 | + if doc.get("schema") != "p037-dispatch-side-report/1": |
| 68 | + bad.append("schema") |
| 69 | + if doc.get("measurement_only") is not True or doc.get("observed_targets_are_exhaustive") \ |
| 70 | + is not False: |
| 71 | + bad.append("the report must say measurement_only and not exhaustive") |
| 72 | + if set(doc) != {"schema", "measurement_only", "observed_targets_are_exhaustive", "calls"}: |
| 73 | + bad.append(f"unexpected top-level keys {sorted(set(doc))}") |
| 74 | + for c in doc.get("calls") or []: |
| 75 | + if not isinstance(c, dict) or set(c) != SIDE_KEYS: |
| 76 | + bad.append(f"call keys {sorted(c) if isinstance(c, dict) else c!r}") |
| 77 | + elif c["dispatch"] not in ("exact", "open"): |
| 78 | + bad.append(f"dispatch {c['dispatch']!r}") |
| 79 | + elif c["dispatch"] == "exact" and c["observed_targets"]: |
| 80 | + bad.append("an exact call carries observed targets") |
| 81 | + return bad |
| 82 | + |
| 83 | + |
| 84 | +def documents(tree: Path, commit: str) -> list[tuple[str, str, list[str]]]: |
| 85 | + """(population, document, files) in §E order, from the tree at `commit`.""" |
| 86 | + def cs(*roots: str) -> list[str]: |
| 87 | + out = git("ls-tree", "-r", "--name-only", commit, "--", *roots).split() |
| 88 | + return sorted(p for p in out if p.endswith(".cs")) |
| 89 | + docs = [("repo", "frontend+audit", cs("frontend", "audit"))] |
| 90 | + docs += [("corpus", f, [f]) for f in cs(*CORPUS)] |
| 91 | + shapes = sorted({p.split("/")[2] for p in cs("corpus/p037-shapes")}) |
| 92 | + docs += [("p037-shapes", s, cs(f"corpus/p037-shapes/{s}")) for s in shapes] |
| 93 | + docs.append(("b0-probes", "probes", cs("docs/evidence/p037-b0-probes"))) |
| 94 | + for _, _, files in docs: |
| 95 | + for f in files: |
| 96 | + if not (tree / f).is_file(): |
| 97 | + raise Refused(f"{f} is not materialized") |
| 98 | + return docs |
| 99 | + |
| 100 | + |
| 101 | +def run_document(tree: Path, files: list[str], work: Path) -> tuple[Any, Any, Any]: |
| 102 | + facts, side = work / "facts.json", work / "dispatch.json" |
| 103 | + cmd = ["dotnet", str(EXTRACTOR), *files, "--flow-locals", "-o", str(facts), |
| 104 | + "--dispatch-report", str(side)] |
| 105 | + x = subprocess.run(cmd, cwd=tree, capture_output=True, text=True, check=False, |
| 106 | + env=ev.sanitized_env()) |
| 107 | + if x.returncode != 0 or ev.reference_contamination(x.stderr): |
| 108 | + raise Refused(f"extractor failed on {files[:2]}: {x.stderr.strip()[-400:]}") |
| 109 | + rep = subprocess.run([str(REPORT_BIN)], input=facts.read_bytes(), capture_output=True, |
| 110 | + check=False) |
| 111 | + if rep.returncode != 0: |
| 112 | + raise Refused(f"own-guarded-report failed: {rep.stderr.decode()[-400:]}") |
| 113 | + return (json.loads(facts.read_text(encoding="utf-8")), json.loads(rep.stdout), |
| 114 | + json.loads(side.read_text(encoding="utf-8"))) |
| 115 | + |
| 116 | + |
| 117 | +def folded(fn: dict[str, Any], var: str, line: int) -> bool: |
| 118 | + """The legacy body releases `var` at a line where a sidecar call takes it.""" |
| 119 | + ops, stack = [], list(fn.get("body") or []) |
| 120 | + while stack: |
| 121 | + op = stack.pop() |
| 122 | + ops.append(op) |
| 123 | + stack += [*op.get("then", []), *op.get("else", []), *op.get("body", [])] |
| 124 | + return any(o.get("op") == "release" and o.get("var") == var and o.get("line") == line |
| 125 | + for o in ops) |
| 126 | + |
| 127 | + |
| 128 | +def analyse(docs: list[tuple[str, str, Any, Any, Any]]) -> dict[str, Any]: |
| 129 | + count: dict[str, collections.Counter[str]] = collections.defaultdict(collections.Counter) |
| 130 | + unexplained: list[dict[str, Any]] = [] |
| 131 | + a14: collections.Counter[str] = collections.Counter() |
| 132 | + r_callees: set[str] = set() |
| 133 | + for pop, name, facts, rep, side in docs: |
| 134 | + fns = {f["name"]: f for f in facts.get("functions", [])} |
| 135 | + coords = {(s["method"], s["ordinal"]): s for s in rep["summary"]} |
| 136 | + for s in rep["summary"]: |
| 137 | + count[f"summary/{pop}"][s["class"]] += 1 |
| 138 | + count["summary"][s["class"]] += 1 |
| 139 | + if s.get("reason"): |
| 140 | + count["nge/summary"][s["reason"]] += 1 |
| 141 | + if s["class"] == "UNEXPLAINED": |
| 142 | + calls = (fns[s["method"]].get("guarded_facts") or {}).get("calls", []) |
| 143 | + lines = [c["statement_line"] for c in calls |
| 144 | + if any(a.get("source_param") == s["ordinal"] for a in c["args"])] |
| 145 | + fold = any(folded(fns[s["method"]], s["param"], ln) for ln in lines) |
| 146 | + unexplained.append({"level": "summary", "doc": name, **s, "legacy_fold": fold}) |
| 147 | + for a in rep["application"]: |
| 148 | + count[f"application/{pop}"][a["class"]] += 1 |
| 149 | + count["application"][a["class"]] += 1 |
| 150 | + if a.get("reason"): |
| 151 | + count["nge/application"][a["reason"]] += 1 |
| 152 | + if a.get("reason") == "callee_no_record" and a["first_party"]: |
| 153 | + r_callees.add(a["callee"]) |
| 154 | + if a["class"] == "UNEXPLAINED": |
| 155 | + var = a["handle"].split(":", 1)[1] if a["handle"].startswith("var:") else \ |
| 156 | + fns[a["caller"]]["params"][int(a["handle"].split(":")[1])]["name"] |
| 157 | + fold = folded(fns[a["caller"]], var, a["statement_line"]) |
| 158 | + unexplained.append({"level": "application", "doc": name, **a, "legacy_fold": fold}) |
| 159 | + sites = collections.defaultdict(list) |
| 160 | + for a in rep["application"]: |
| 161 | + sites[(a["caller"], a["site"]["line"], a["site"]["column"])].append(a) |
| 162 | + for c in side["calls"]: |
| 163 | + a14["relevant_calls"] += 1 |
| 164 | + a14[c["dispatch"]] += 1 |
| 165 | + rows = sites.get((c["caller"], c["site"]["line"], c["site"]["column"]), []) |
| 166 | + if c["dispatch"] != "open": |
| 167 | + continue |
| 168 | + a14["open_in_guarded_chain"] += any(r["in_chain"] for r in rows) |
| 169 | + values = [] |
| 170 | + for slot in {r["slot"] for r in rows}: |
| 171 | + targets = [coords.get((t, slot)) for t in [c["callee"], *c["observed_targets"]]] |
| 172 | + values.append([(x.get("guarded"), x.get("shape"), str(x.get("cells"))) |
| 173 | + if x else None for x in targets]) |
| 174 | + a14["open_observed_targets_differ"] += any(len(set(v)) > 1 for v in values) |
| 175 | + a14["kill_witness_static_must_observed_not"] += any( |
| 176 | + v[0] is not None and v[0][0] == "must" |
| 177 | + and any(x is None or x[0] != "must" for x in v[1:]) for v in values) |
| 178 | + r_direct = count["nge/summary"]["callee_no_record"] |
| 179 | + r_via = count["nge/summary"]["via:callee_no_record"] |
| 180 | + eligible = count["summary"]["EQUAL"] + count["summary"]["SUMMARY_REFINEMENT"] + \ |
| 181 | + count["summary"]["LEGACY_HONESTY"] + count["summary"]["UNEXPLAINED"] + r_direct + r_via |
| 182 | + return { |
| 183 | + "counts": {k: dict(sorted(v.items())) for k, v in sorted(count.items())}, |
| 184 | + "unexplained": unexplained, |
| 185 | + "a14": dict(a14), |
| 186 | + "r": {"first_party_callees_without_record": sorted(r_callees), |
| 187 | + "coordinates_ending_at_r": r_direct, "coordinates_tainted_by_r": r_via, |
| 188 | + "share_of_eligible_chains_broken_by_r": |
| 189 | + round((r_direct + r_via) / eligible, 4) if eligible else None}, |
| 190 | + } |
| 191 | + |
| 192 | + |
| 193 | +def sha(p: Path) -> str: |
| 194 | + return hashlib.sha256(p.read_bytes()).hexdigest() |
| 195 | + |
| 196 | + |
| 197 | +def main(argv: list[str]) -> int: |
| 198 | + ap = argparse.ArgumentParser(description=__doc__, |
| 199 | + formatter_class=argparse.RawDescriptionHelpFormatter) |
| 200 | + ap.add_argument("--commit", default=POPULATION_COMMIT) |
| 201 | + ap.add_argument("--out", required=True) |
| 202 | + args = ap.parse_args(argv) |
| 203 | + try: |
| 204 | + commit = frozen(args.commit) |
| 205 | + if git("status", "--porcelain", "--untracked-files=no").strip(): |
| 206 | + raise Refused("the instrument tree is dirty; commit it first") |
| 207 | + for p in (EXTRACTOR, REPORT_BIN): |
| 208 | + if not p.is_file(): |
| 209 | + raise Refused(f"{p} is not built") |
| 210 | + with tempfile.TemporaryDirectory(prefix="p037-b1-") as td: |
| 211 | + tree = Path(td) / "tree" |
| 212 | + tree.mkdir() |
| 213 | + archive = subprocess.run(["git", "archive", commit], cwd=ROOT, capture_output=True, |
| 214 | + check=True).stdout |
| 215 | + subprocess.run(["tar", "-x", "-C", str(tree)], input=archive, check=True) |
| 216 | + rows = [] |
| 217 | + for pop, name, files in documents(tree, commit): |
| 218 | + facts, rep, side = run_document(tree, files, Path(td)) |
| 219 | + if problems := validate_side_report(side): |
| 220 | + raise Refused(f"{name}: side report {problems}") |
| 221 | + if rep.get("static_dispatch_conditional") is not True: |
| 222 | + raise Refused(f"{name}: report is not static-dispatch conditional (A14)") |
| 223 | + rows.append((pop, name, facts, rep, side)) |
| 224 | + print(f" {pop}/{name}: {len(rep['summary'])} coordinates, " |
| 225 | + f"{len(rep['application'])} sites", file=sys.stderr) |
| 226 | + except Refused as exc: |
| 227 | + print(f"REFUSED: {exc}") |
| 228 | + return 2 |
| 229 | + result = analyse(rows) |
| 230 | + controls = [r for r in result["unexplained"] if any(c in r["doc"] for c in CONTROLS)] |
| 231 | + out = { |
| 232 | + "schema": "p037-b1-shadow-run/1", |
| 233 | + "population_commit": commit, |
| 234 | + "instrument_commit": git("rev-parse", "HEAD").strip(), |
| 235 | + "toolchain": {"dotnet": git_free(["dotnet", "--version"]), |
| 236 | + "rustc": git_free(["rustc", "--version"])}, |
| 237 | + "instrument": {"extractor_sha256": sha(EXTRACTOR), "report_sha256": sha(REPORT_BIN)}, |
| 238 | + "documents": {pop: sum(1 for r in rows if r[0] == pop) for pop in dict.fromkeys( |
| 239 | + r[0] for r in rows)}, |
| 240 | + "static_dispatch_conditional": True, |
| 241 | + **result, |
| 242 | + "unexplained_in_p037_controls": len(controls), |
| 243 | + } |
| 244 | + Path(args.out).write_text(json.dumps(out, indent=1, sort_keys=False) + "\n", encoding="utf-8") |
| 245 | + unexplained = sum(1 for _ in result["unexplained"]) |
| 246 | + print(json.dumps({k: out[k] for k in ("documents", "counts", "a14", "r")}, indent=1)) |
| 247 | + print(f"RESULT: {'FAIL' if unexplained else 'PASS'}: UNEXPLAINED = {unexplained}") |
| 248 | + return 1 if unexplained else 0 |
| 249 | + |
| 250 | + |
| 251 | +def git_free(argv: list[str]) -> str: |
| 252 | + return subprocess.run(argv, capture_output=True, text=True, check=False).stdout.strip() |
| 253 | + |
| 254 | + |
| 255 | +if __name__ == "__main__": |
| 256 | + sys.exit(main(sys.argv[1:])) |
0 commit comments