Skip to content

Commit 89cec34

Browse files
TMHSDigitalclaude
andauthored
fix(scripts): measure_hero_drift exits non-zero on size mismatch, unknown --only, empty run (#315)
- a size mismatch between fresh render and committed hero is a failure (exit 3) - an unknown --only name or a run that measured nothing exits 2 with a message - a failing `blender --version` probe or unrunnable binary exits 2 instead of IndexError - a bare trailing --output in a README command is handled, not an IndexError - unit tests (no Blender) run in the validate-harness job Closes #296 Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
1 parent 0dbb94b commit 89cec34

3 files changed

Lines changed: 135 additions & 5 deletions

File tree

‎.github/workflows/validate.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -385,3 +385,8 @@ jobs:
385385

386386
- name: Harness unit tests
387387
run: python3 tests/smoke/test_harness.py -v
388+
389+
- name: Hero-drift tool exit-code tests
390+
run: |
391+
pip install numpy==2.2.6 Pillow==11.2.1
392+
python3 tests/test_measure_hero_drift.py -v

‎scripts/measure_hero_drift.py‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,9 @@
2626
python scripts/measure_hero_drift.py --blender PATH [--only NAME ...]
2727
[--out DIR] [--json FILE]
2828
29-
Exit codes: 0 measured (drift is reported, not judged), 2 usage, 3 a render
30-
failed for at least one entry.
29+
Exit codes: 0 measured (drift is reported, not judged), 2 usage (unknown
30+
--only name, nothing measured, unreadable Blender version), 3 a render failed or
31+
a fresh render's size differs from the committed hero (a size change is drift).
3132
"""
3233
from __future__ import annotations
3334

@@ -76,6 +77,8 @@ def render_args(entry: dict, png: Path) -> list[str] | None:
7677
continue
7778
args = shlex.split(m.group(1))
7879
i = args.index("--output")
80+
if i + 1 >= len(args):
81+
return None # bare trailing --output: no path to retarget
7982
args[i + 1] = str(png)
8083
return args
8184
return None
@@ -124,17 +127,34 @@ def main(argv=None) -> int:
124127
p.add_argument("--json", default=None, help="also write the results as JSON here")
125128
args = p.parse_args(argv)
126129

130+
all_entries = entries()
131+
unknown = sorted(set(args.only or []) - {e["name"] for e in all_entries})
132+
if unknown:
133+
print(f"error: --only names match no example or showcase piece: {', '.join(unknown)}",
134+
file=sys.stderr)
135+
return 2
136+
127137
blender = str(Path(args.blender).resolve())
128138
out_dir = Path(args.out)
129139
out_dir.mkdir(parents=True, exist_ok=True)
130-
version = subprocess.run([blender, "--version"], capture_output=True, text=True,
131-
encoding="utf-8", errors="replace").stdout.splitlines()[0]
140+
try:
141+
probe = subprocess.run([blender, "--version"], capture_output=True, text=True,
142+
encoding="utf-8", errors="replace")
143+
except OSError as exc:
144+
print(f"error: cannot run Blender at {blender}: {exc}", file=sys.stderr)
145+
return 2
146+
lines = probe.stdout.splitlines()
147+
if probe.returncode != 0 or not lines:
148+
print(f"error: `{blender} --version` failed (exit {probe.returncode}) or printed nothing",
149+
file=sys.stderr)
150+
return 2
151+
version = lines[0]
132152
print(f"# renderer: {blender}\n# version: {version}", flush=True)
133153
print(f"{'name':32} {'mean_abs':>9} {'gt2pct':>8} {'vs_q90':>8} {'luma c/f':>17} {'secs':>6} verdict",
134154
flush=True)
135155

136156
results, failed = [], []
137-
for e in entries():
157+
for e in all_entries:
138158
name = e["name"]
139159
if args.only and name not in args.only:
140160
continue
@@ -162,6 +182,7 @@ def main(argv=None) -> int:
162182
results.append(row)
163183
if "mean_abs" not in row:
164184
print(f"{name:32} size mismatch {row}", flush=True)
185+
failed.append(name)
165186
continue
166187
verdict = "matches" if row["vs_q90"] <= DRIFT_THRESHOLD else "DRIFTED"
167188
row["verdict"] = verdict
@@ -175,6 +196,9 @@ def main(argv=None) -> int:
175196
encoding="utf-8")
176197
drifted = [r["name"] for r in results if r.get("verdict") == "DRIFTED"]
177198
print(f"\n# {len(results)} measured, {len(drifted)} drifted, {len(failed)} failed")
199+
if not results and not failed:
200+
print("error: no heroes were measured", file=sys.stderr)
201+
return 2
178202
return 3 if failed else 0
179203

180204

‎tests/test_measure_hero_drift.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
"""Exit-code tests for scripts/measure_hero_drift.py. No Blender required.
2+
3+
Needs numpy and Pillow (the script's own dependencies). Run:
4+
python tests/test_measure_hero_drift.py -v
5+
"""
6+
from __future__ import annotations
7+
8+
import contextlib
9+
import io
10+
import sys
11+
import tempfile
12+
import unittest
13+
from pathlib import Path
14+
from unittest import mock
15+
16+
REPO = Path(__file__).resolve().parent.parent
17+
sys.path.insert(0, str(REPO / "scripts"))
18+
19+
import measure_hero_drift as m # noqa: E402
20+
21+
ENTRY = {"name": "demo", "dir": "examples/demo", "hero": "docs/gallery/assets/demo-hero.webp"}
22+
23+
24+
def run(argv, *, entries=(ENTRY,), version="Blender 5.2.1 LTS", version_exit=0, size_match=True):
25+
"""Call m.main with every external effect faked; return (exit code, stderr)."""
26+
td = Path(tempfile.mkdtemp())
27+
(td / "examples" / "demo").mkdir(parents=True)
28+
(td / "examples" / "demo" / "demo.py").write_text("", encoding="utf-8")
29+
30+
def fake_run(command, **kw):
31+
if "--version" in command:
32+
return mock.Mock(stdout=(version + "\n") if version else "", returncode=version_exit)
33+
Path(command[command.index("--output") + 1]).write_bytes(b"png")
34+
return mock.Mock(returncode=0, stdout="", stderr="")
35+
36+
def fake_compare(committed, fresh):
37+
if not size_match:
38+
return {"shape_committed": (16, 16), "shape_fresh": (8, 8)}
39+
return {"mean_abs": 0.0, "gt2pct": 0.0, "vs_q90": 0.0,
40+
"luma_committed": 0.1, "luma_fresh": 0.1, "bytes_committed": 1}
41+
42+
err = io.StringIO()
43+
with mock.patch.object(m, "REPO", td), \
44+
mock.patch.object(m, "entries", lambda: list(entries)), \
45+
mock.patch.object(m, "render_args", lambda e, png: ["--output", str(png)]), \
46+
mock.patch.object(m.subprocess, "run", fake_run), \
47+
mock.patch.object(m, "compare", fake_compare), \
48+
contextlib.redirect_stderr(err), contextlib.redirect_stdout(io.StringIO()):
49+
code = m.main(["--blender", "blender", "--out", str(td / "out"), *argv])
50+
return code, err.getvalue()
51+
52+
53+
class ExitCodes(unittest.TestCase):
54+
def test_matching_sizes_exit_0(self):
55+
self.assertEqual(run([])[0], 0)
56+
57+
def test_size_mismatch_is_a_failure(self):
58+
code, _ = run([], size_match=False)
59+
self.assertEqual(code, 3)
60+
61+
def test_unknown_only_name_is_usage_error(self):
62+
code, err = run(["--only", "nope"])
63+
self.assertEqual(code, 2)
64+
self.assertIn("nope", err)
65+
66+
def test_nothing_measured_is_usage_error(self):
67+
self.assertEqual(run([], entries=())[0], 2)
68+
69+
def test_blender_version_failure_is_clear_error(self):
70+
code, err = run([], version="", version_exit=1)
71+
self.assertEqual(code, 2)
72+
self.assertIn("--version", err)
73+
74+
75+
class RenderArgs(unittest.TestCase):
76+
def test_bare_trailing_output_returns_none(self):
77+
with tempfile.TemporaryDirectory() as td:
78+
d = Path(td) / "examples" / "demo"
79+
d.mkdir(parents=True)
80+
(d / "README.md").write_text(
81+
"blender --background --python demo.py -- --engine cycles --output\n",
82+
encoding="utf-8",
83+
)
84+
with mock.patch.object(m, "REPO", Path(td)):
85+
self.assertIsNone(m.render_args({"dir": "examples/demo"}, Path("p.png")))
86+
87+
def test_output_path_is_retargeted(self):
88+
with tempfile.TemporaryDirectory() as td:
89+
d = Path(td) / "examples" / "demo"
90+
d.mkdir(parents=True)
91+
(d / "README.md").write_text(
92+
"blender --background --python demo.py -- --output out.png --engine cycles\n",
93+
encoding="utf-8",
94+
)
95+
with mock.patch.object(m, "REPO", Path(td)):
96+
args = m.render_args({"dir": "examples/demo"}, Path("fresh.png"))
97+
self.assertEqual(args[args.index("--output") + 1], "fresh.png")
98+
99+
100+
if __name__ == "__main__":
101+
unittest.main()

0 commit comments

Comments
 (0)