Skip to content

Commit 29edf01

Browse files
TMHSDigitalclaude
andauthored
chore(scripts): add measure_hero_drift.py for issue #200 (#218)
Renders every gallery entry's README --output command to lossless PNG and compares it with the committed hero, using #200's metric (it reproduces that issue's 0.01683 / 22.45% for shipping-crate and 0.00330 / 0.50% for crate-stack exactly). One correction to the method: renders are deterministic, but the webp encode floor is not a constant (0.1-4% of pixels depending on the image), so a fixed threshold on the PNG comparison misclassifies. The verdict compares against the fresh render re-encoded at webp q90 instead; on the full 79-entry sweep that splits the tree with no overlap (matches <= 1.8%, drifted >= 3.8%). Authoring tool only; not wired into CI. Refs #200 Signed-off-by: TMHSDigital <154358121+TMHSDigital@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1 parent b0bc6f2 commit 29edf01

1 file changed

Lines changed: 182 additions & 0 deletions

File tree

‎scripts/measure_hero_drift.py‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
#!/usr/bin/env python3
2+
"""Measure how far each committed gallery hero is from what its script renders.
3+
4+
For every entry in ``examples/gallery.json`` and ``showcase/gallery.json``,
5+
render the entry's README ``--output`` command to a lossless PNG with the
6+
given Blender binary, then compare it with the committed
7+
``docs/gallery/assets/<name>-hero.webp``:
8+
9+
* ``mean_abs`` - mean absolute per-channel difference, 0..1
10+
* ``gt2pct`` - share of pixels whose mean channel difference exceeds 2%
11+
* ``vs_q90`` - ``gt2pct`` against the fresh render re-encoded as webp q90
12+
* ``luma`` - mean luminance (Rec. 709) of the committed and fresh image
13+
14+
This is the method from issue #200, with one correction. Renders are
15+
deterministic (a re-render is pixel-identical), but the webp encode floor is
16+
not a constant: re-encoding a frame at q90 alone moves 0.1-4% of pixels past
17+
2%, depending on the image. ``gt2pct`` against the PNG therefore mixes drift
18+
with encode noise. The verdict uses ``vs_q90``, which cancels the encode: on
19+
the 2026-09-23 sweep every matching hero scored <= 1.8% and every drifted one
20+
>= 3.8%, hence DRIFT_THRESHOLD.
21+
22+
Authoring tool, not a CI step: one full render per entry. Requires Pillow
23+
and numpy on the host Python (not Blender's).
24+
25+
Usage:
26+
python scripts/measure_hero_drift.py --blender PATH [--only NAME ...]
27+
[--out DIR] [--json FILE]
28+
29+
Exit codes: 0 measured (drift is reported, not judged), 2 usage, 3 a render
30+
failed for at least one entry.
31+
"""
32+
from __future__ import annotations
33+
34+
import argparse
35+
import io
36+
import json
37+
import re
38+
import shlex
39+
import subprocess
40+
import sys
41+
import time
42+
from pathlib import Path
43+
44+
import numpy as np
45+
from PIL import Image
46+
47+
REPO = Path(__file__).resolve().parent.parent
48+
SOURCES = ("examples/gallery.json", "showcase/gallery.json")
49+
DRIFT_THRESHOLD = 0.025 # vs_q90 share; the gap on the full sweep is 1.8% .. 3.8%
50+
51+
_RUN_LINE = re.compile(r"blender\s+--background\s+--python\s+\S+\.py\s+--\s+(.*--output\s.*)$")
52+
53+
54+
def entries() -> list[dict]:
55+
out = []
56+
for rel in SOURCES:
57+
data = json.loads((REPO / rel).read_text(encoding="utf-8"))
58+
items = next(v for v in data.values() if isinstance(v, list))
59+
out.extend(items)
60+
return out
61+
62+
63+
def render_args(entry: dict, png: Path) -> list[str] | None:
64+
"""The README's documented ``--output`` invocation, retargeted at *png*.
65+
66+
Returns None when the README documents no render command. Only the
67+
output path is replaced; every other documented flag (``--engine``,
68+
``--samples`` ...) is kept, because the hero was rendered with them.
69+
"""
70+
readme = REPO / entry["dir"] / "README.md"
71+
if not readme.is_file():
72+
return None
73+
for line in readme.read_text(encoding="utf-8").splitlines():
74+
m = _RUN_LINE.search(line.strip())
75+
if not m:
76+
continue
77+
args = shlex.split(m.group(1))
78+
i = args.index("--output")
79+
args[i + 1] = str(png)
80+
return args
81+
return None
82+
83+
84+
def load(path: Path) -> np.ndarray:
85+
return np.asarray(Image.open(path).convert("RGB"), dtype=np.float64) / 255.0
86+
87+
88+
def luma(a: np.ndarray) -> float:
89+
return float((a @ np.array([0.2126, 0.7152, 0.0722])).mean())
90+
91+
92+
def gt2pct(a: np.ndarray, b: np.ndarray) -> float:
93+
# Mean over the channels, not any channel: the definition #200 used,
94+
# which reproduces its 22.45% (shipping-crate) and 0.50% (crate-stack).
95+
return round(float((np.abs(a - b).mean(axis=2) > 0.02).mean()), 5)
96+
97+
98+
def webp_q90(path: Path) -> np.ndarray:
99+
buf = io.BytesIO()
100+
Image.open(path).convert("RGB").save(buf, "WEBP", quality=90)
101+
return np.asarray(Image.open(buf).convert("RGB"), dtype=np.float64) / 255.0
102+
103+
104+
def compare(committed: Path, fresh: Path) -> dict:
105+
a, b = load(committed), load(fresh)
106+
if a.shape != b.shape:
107+
return {"shape_committed": a.shape[:2], "shape_fresh": b.shape[:2]}
108+
return {
109+
"mean_abs": round(float(np.abs(a - b).mean()), 5),
110+
"gt2pct": gt2pct(a, b),
111+
"vs_q90": gt2pct(a, webp_q90(fresh)),
112+
"luma_committed": round(luma(a), 5),
113+
"luma_fresh": round(luma(b), 5),
114+
"bytes_committed": committed.stat().st_size,
115+
}
116+
117+
118+
def main(argv=None) -> int:
119+
p = argparse.ArgumentParser(description=__doc__.split("\n")[0])
120+
p.add_argument("--blender", required=True, help="Blender binary to render with")
121+
p.add_argument("--only", action="append", help="measure only this entry (repeatable)")
122+
p.add_argument("--out", default=str(REPO / ".scratch" / "hero-drift"),
123+
help="directory for fresh PNG renders (default: .scratch/hero-drift)")
124+
p.add_argument("--json", default=None, help="also write the results as JSON here")
125+
args = p.parse_args(argv)
126+
127+
blender = str(Path(args.blender).resolve())
128+
out_dir = Path(args.out)
129+
out_dir.mkdir(parents=True, exist_ok=True)
130+
version = subprocess.run([blender, "--version"], capture_output=True, text=True,
131+
encoding="utf-8", errors="replace").stdout.splitlines()[0]
132+
print(f"# renderer: {blender}\n# version: {version}", flush=True)
133+
print(f"{'name':32} {'mean_abs':>9} {'gt2pct':>8} {'vs_q90':>8} {'luma c/f':>17} {'secs':>6} verdict",
134+
flush=True)
135+
136+
results, failed = [], []
137+
for e in entries():
138+
name = e["name"]
139+
if args.only and name not in args.only:
140+
continue
141+
png = out_dir / f"{name}.png"
142+
cmd = render_args(e, png)
143+
if cmd is None:
144+
print(f"{name:32} no --output command in README", flush=True)
145+
failed.append(name)
146+
continue
147+
script = next((REPO / e["dir"]).glob("*.py"))
148+
if png.exists():
149+
png.unlink()
150+
t0 = time.time()
151+
proc = subprocess.run(
152+
[blender, "--background", "--factory-startup", "--python", str(script), "--", *cmd],
153+
capture_output=True, text=True, encoding="utf-8", errors="replace", cwd=REPO,
154+
)
155+
secs = time.time() - t0
156+
if proc.returncode != 0 or not png.is_file():
157+
print(f"{name:32} render failed: exit {proc.returncode}", flush=True)
158+
failed.append(name)
159+
continue
160+
row = {"name": name, "args": cmd, "secs": round(secs, 1),
161+
**compare(REPO / e["hero"], png)}
162+
results.append(row)
163+
if "mean_abs" not in row:
164+
print(f"{name:32} size mismatch {row}", flush=True)
165+
continue
166+
verdict = "matches" if row["vs_q90"] <= DRIFT_THRESHOLD else "DRIFTED"
167+
row["verdict"] = verdict
168+
print(f"{name:32} {row['mean_abs']:9.5f} {row['gt2pct']:8.2%} {row['vs_q90']:8.2%} "
169+
f"{row['luma_committed']:.4f}/{row['luma_fresh']:.4f} {secs:6.1f} {verdict}",
170+
flush=True)
171+
172+
if args.json:
173+
Path(args.json).write_text(json.dumps(
174+
{"renderer": version, "results": results, "failed": failed}, indent=2),
175+
encoding="utf-8")
176+
drifted = [r["name"] for r in results if r.get("verdict") == "DRIFTED"]
177+
print(f"\n# {len(results)} measured, {len(drifted)} drifted, {len(failed)} failed")
178+
return 3 if failed else 0
179+
180+
181+
if __name__ == "__main__":
182+
sys.exit(main())

0 commit comments

Comments
 (0)