Skip to content

Commit a7017a1

Browse files
TMHSDigitalclaude
andcommitted
test: self-tests for the Validate gates, so a regressed gate turns red
Only check_exit_code_readme had tests; every other gate ran only against a clean repo, where a gate that stopped gating still passes. tests/test_gates.py builds failing fixtures for check_smoke_catalog (unknown keys, forbidden exits, expectation rules, coverage, missing script), check_content_catalog (uncatalogued snippet, bad falsify pairs, non-compiling skill block line), check_counts, check_example_rules and check_falsifier_targets' static mode. Falsified: inverting the forbidden-exit test in check_smoke_catalog fails 6. check_content_catalog gains a check(root, cases) seam for this; check_smoke_catalog now also rejects a row whose script file is missing (previously only Blender noticed at runtime); check_falsifier_targets closes the files it reads. Closes #527 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LpkeXRvikhJZSVGBFM91Dv Signed-off-by: TM Hospitality Strategies <TMHospitalityStrategies@gmail.com>
1 parent 24608fc commit a7017a1

5 files changed

Lines changed: 287 additions & 11 deletions

File tree

‎.github/workflows/validate.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -442,3 +442,6 @@ jobs:
442442

443443
- name: Hero-drift tool exit-code tests
444444
run: python3 tests/test_measure_hero_drift.py -v
445+
446+
- name: Validate gate self-tests (a gate that regresses fails here, #527)
447+
run: python3 tests/test_gates.py -v

‎tests/check_content_catalog.py‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,17 @@
2323
TAG = re.compile(r"^<!-- smoke: run -->[ \t]*\r?\n```python", re.M)
2424

2525

26-
def main():
26+
def check(root=ROOT, cases=None):
27+
"""(errors, skill python blocks, tagged blocks) for a repo root."""
28+
cases = CASES if cases is None else cases
2729
errors = []
28-
snippets = {os.path.basename(p) for p in glob.glob(os.path.join(ROOT, "snippets", "*.py"))}
29-
for name in sorted(snippets - set(CASES)):
30+
snippets = {os.path.basename(p) for p in glob.glob(os.path.join(root, "snippets", "*.py"))}
31+
for name in sorted(snippets - set(cases)):
3032
errors.append(f"snippets/{name} has no entry in tests/smoke/snippet_checks.py")
31-
for name in sorted(set(CASES) - snippets):
33+
for name in sorted(set(cases) - snippets):
3234
errors.append(f"tests/smoke/snippet_checks.py names snippets/{name}, which does not exist")
33-
for name in sorted(set(CASES) & snippets):
34-
case = CASES[name]
35+
for name in sorted(set(cases) & snippets):
36+
case = cases[name]
3537
unknown = set(case) - {"setup", "check", "falsify"}
3638
if unknown:
3739
errors.append(f"{name}: unknown keys {sorted(unknown)}")
@@ -41,13 +43,13 @@ def main():
4143
if not (isinstance(fals, tuple) and len(fals) == 2 and fals[0] != fals[1]):
4244
errors.append(f"{name}: falsify must be an (old, new) pair that changes something")
4345
continue
44-
with open(os.path.join(ROOT, "snippets", name), encoding="utf-8") as fh:
46+
with open(os.path.join(root, "snippets", name), encoding="utf-8") as fh:
4547
if fals[0] not in fh.read():
4648
errors.append(f"{name}: falsify string {fals[0]!r} is not in the snippet")
4749

4850
blocks = tagged = 0
49-
for path in sorted(glob.glob(os.path.join(ROOT, "skills", "*", "SKILL.md"))):
50-
rel = os.path.relpath(path, ROOT).replace(os.sep, "/")
51+
for path in sorted(glob.glob(os.path.join(root, "skills", "*", "SKILL.md"))):
52+
rel = os.path.relpath(path, root).replace(os.sep, "/")
5153
with open(path, encoding="utf-8") as fh:
5254
text = fh.read()
5355
tagged += len(TAG.findall(text))
@@ -58,7 +60,11 @@ def main():
5860
compile(m.group(1), rel, "exec")
5961
except SyntaxError as e:
6062
errors.append(f"{rel}:{start + (e.lineno or 1) - 1}: python block does not compile: {e.msg}")
63+
return errors, blocks, tagged
64+
6165

66+
def main():
67+
errors, blocks, tagged = check()
6268
for e in errors:
6369
print(f"::error::{e}")
6470
if errors:

‎tests/check_falsifier_targets.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,8 @@ def _column(headers, *patterns):
108108

109109
def parse_readme(path):
110110
"""(falsifiers {flag: (exit, target)}, exit_table {code: meaning})."""
111-
text = io.open(path, encoding="utf-8").read()
111+
with io.open(path, encoding="utf-8") as fh:
112+
text = fh.read()
112113
falsifiers, exits = {}, {}
113114
for headers, rows in _tables(text):
114115
code_col = _column(headers, r"^exit$", r"exit code", r"^code$")
@@ -152,7 +153,8 @@ def parse_readme(path):
152153

153154
def script_flags(path):
154155
"""store_true argparse flags declared in the script."""
155-
src = io.open(path, encoding="utf-8").read()
156+
with io.open(path, encoding="utf-8") as fh:
157+
src = fh.read()
156158
flags = set()
157159
try:
158160
tree = ast.parse(src)

‎tests/check_smoke_catalog.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,12 @@ def check(root: Path) -> list[str]:
103103
errors: list[str] = []
104104
for row in catalog:
105105
errors += check_schema(row)
106+
if not isinstance(row.get("script"), str):
107+
continue
106108
errors += check_readme_claims(root, row)
109+
if not (root / row["script"]).is_file():
110+
# otherwise only Blender, at runtime, notices a renamed script (#527)
111+
errors.append(f"catalog row {row.get('name')!r}: script {row['script']} does not exist")
107112
parts = Path(row["script"]).parts
108113
if len(parts) >= 2 and parts[0] in ("examples", "showcase"):
109114
key = f"{parts[0]}/{parts[1]}"

‎tests/test_gates.py‎

Lines changed: 260 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,260 @@
1+
"""Self-tests for the Validate gate scripts (#527). No Blender required.
2+
3+
A gate with a bug passes everything, and nothing would notice: Validate only
4+
ever runs the gates against a repo that is (supposed to be) clean. Each test
5+
here builds a small fixture that a gate must reject, so an inverted condition
6+
or a typo in a key set turns this file red instead.
7+
8+
python tests/test_gates.py -v
9+
"""
10+
from __future__ import annotations
11+
12+
import json
13+
import sys
14+
import tempfile
15+
import unittest
16+
from pathlib import Path
17+
from unittest import mock
18+
19+
TESTS = Path(__file__).resolve().parent
20+
sys.path.insert(0, str(TESTS))
21+
sys.path.insert(0, str(TESTS / "smoke"))
22+
23+
import check_content_catalog as ccc # noqa: E402
24+
import check_counts # noqa: E402
25+
import check_example_rules as cer # noqa: E402
26+
import check_falsifier_targets as cft # noqa: E402
27+
import check_smoke_catalog as csc # noqa: E402
28+
29+
30+
def write(root: Path, rel: str, text: str) -> Path:
31+
p = root / rel
32+
p.parent.mkdir(parents=True, exist_ok=True)
33+
p.write_text(text, encoding="utf-8")
34+
return p
35+
36+
37+
GOOD_FALSIFIER = {"args": ["--break"], "expect_exit": 3}
38+
39+
40+
class SmokeCatalogSchema(unittest.TestCase):
41+
def errs(self, **row):
42+
base = {"name": "x", "script": "examples/x/x.py", "falsifiers": [GOOD_FALSIFIER]}
43+
base.update(row)
44+
return csc.check_schema({k: v for k, v in base.items() if v is not None})
45+
46+
def test_good_row_passes(self):
47+
self.assertEqual(self.errs(), [])
48+
49+
def test_unknown_row_key(self):
50+
self.assertTrue(any("unknown key 'falsifer'" in e for e in self.errs(falsifer=[])))
51+
52+
def test_forbidden_exits(self):
53+
for bad in (0, 1, 2, 77, True):
54+
with self.subTest(bad=bad):
55+
e = self.errs(falsifiers=[{"args": ["--x"], "expect_exit": bad}])
56+
self.assertTrue(any("expect_exit" in m for m in e), e)
57+
58+
def test_exactly_one_expectation(self):
59+
e = self.errs(expect_sidecar="o", falsifiers=[
60+
{"args": ["--x"], "expect_exit": 3, "expect_sidecar_fail": True}])
61+
self.assertTrue(any("exactly one" in m for m in e), e)
62+
63+
def test_sidecar_fail_needs_a_sidecar(self):
64+
e = self.errs(falsifiers=[{"args": ["--x"], "expect_sidecar_fail": True}])
65+
self.assertTrue(any("no expect_sidecar" in m for m in e), e)
66+
67+
def test_no_falsifiers(self):
68+
for fz in ([], None):
69+
with self.subTest(fz=fz):
70+
self.assertTrue(any("no falsifiers" in m for m in self.errs(falsifiers=fz)))
71+
72+
def test_bad_falsifier_args(self):
73+
e = self.errs(falsifiers=[{"args": [], "expect_exit": 3}])
74+
self.assertTrue(any("'args'" in m for m in e), e)
75+
76+
77+
class SmokeCatalogTree(unittest.TestCase):
78+
def run_check(self, rows, dirs=("examples/x",), scripts=("examples/x/x.py",)):
79+
with tempfile.TemporaryDirectory() as td:
80+
root = Path(td)
81+
(root / "showcase").mkdir()
82+
for d in dirs:
83+
(root / d).mkdir(parents=True)
84+
for s in scripts:
85+
write(root, s, "")
86+
write(root, "tests/smoke/catalog.json", json.dumps(rows))
87+
return csc.check(root)
88+
89+
def row(self, **kw):
90+
r = {"name": "x", "script": "examples/x/x.py", "falsifiers": [GOOD_FALSIFIER]}
91+
r.update(kw)
92+
return r
93+
94+
def test_clean_tree_passes(self):
95+
self.assertEqual(self.run_check([self.row()]), [])
96+
97+
def test_empty_catalog(self):
98+
self.assertEqual(self.run_check([]), ["tests/smoke/catalog.json is empty"])
99+
100+
def test_example_without_row(self):
101+
e = self.run_check([self.row()], dirs=("examples/x", "examples/y"))
102+
self.assertTrue(any("examples/y has no row" in m for m in e), e)
103+
104+
def test_row_for_missing_directory(self):
105+
e = self.run_check([self.row(), self.row(name="z", script="examples/z/z.py")])
106+
self.assertTrue(any("missing directory examples/z" in m for m in e), e)
107+
108+
def test_row_for_missing_script(self):
109+
e = self.run_check([self.row(script="examples/x/renamed.py")])
110+
self.assertTrue(any("renamed.py does not exist" in m for m in e), e)
111+
112+
113+
class ContentCatalog(unittest.TestCase):
114+
def run_check(self, cases, snippet="x = 1\n", skill=None):
115+
with tempfile.TemporaryDirectory() as td:
116+
root = Path(td)
117+
write(root, "snippets/s.py", snippet)
118+
if skill is not None:
119+
write(root, "skills/k/SKILL.md", skill)
120+
return ccc.check(str(root), cases)[0]
121+
122+
def test_clean_passes(self):
123+
self.assertEqual(self.run_check({"s.py": {"check": "pass", "falsify": ("x = 1", "x = 2")}}), [])
124+
125+
def test_snippet_without_case(self):
126+
self.assertTrue(any("no entry" in e for e in self.run_check({})))
127+
128+
def test_falsify_old_not_in_source(self):
129+
e = self.run_check({"s.py": {"check": "pass", "falsify": ("y = 1", "y = 2")}})
130+
self.assertTrue(any("not in the snippet" in m for m in e), e)
131+
132+
def test_falsify_that_changes_nothing(self):
133+
e = self.run_check({"s.py": {"check": "pass", "falsify": ("x = 1", "x = 1")}})
134+
self.assertTrue(any("changes something" in m for m in e), e)
135+
136+
def test_skill_block_that_does_not_compile_reports_its_line(self):
137+
skill = "# K\n\ntext\n\n```python\nok = 1\nif True\n pass\n```\n"
138+
e = self.run_check({"s.py": {"check": "pass", "falsify": ("x = 1", "x = 2")}}, skill=skill)
139+
self.assertTrue(any("skills/k/SKILL.md:7" in m for m in e), e)
140+
141+
142+
class Counts(unittest.TestCase):
143+
def tree(self, readme: str) -> list[str]:
144+
with tempfile.TemporaryDirectory() as td:
145+
root = Path(td)
146+
write(root, "skills/a/SKILL.md", "x")
147+
write(root, "skills/b/SKILL.md", "x")
148+
write(root, "rules/r.mdc", "x")
149+
(root / "templates" / "t").mkdir(parents=True)
150+
write(root, "snippets/s.py", "a\nb\nc\nd\ne\n")
151+
write(root, "examples/e/README.md", "x")
152+
write(root, "examples/gallery.json", json.dumps({"examples": []}))
153+
(root / "showcase").mkdir()
154+
write(root, "README.md", readme)
155+
return check_counts.check(root)
156+
157+
GOOD = ("2 skills, 1 rule, 1 template, 1 snippet, 1 example, 0 showcase pieces. "
158+
"Rules: `r`.\n")
159+
160+
def test_good_readme_passes(self):
161+
self.assertEqual(self.tree(self.GOOD), [])
162+
163+
def test_stale_count_fails(self):
164+
e = self.tree(self.GOOD + "The pack has 3 skills.\n")
165+
self.assertTrue(any('"3 skills"' in m for m in e), e)
166+
167+
def test_unnamed_rule_fails(self):
168+
e = self.tree(self.GOOD.replace("Rules: `r`.", ""))
169+
self.assertTrue(any("never names rule `r`" in m for m in e), e)
170+
171+
def test_snippet_over_the_cap_fails(self):
172+
with tempfile.TemporaryDirectory() as td:
173+
root = Path(td)
174+
for rel, text in (("skills/a/SKILL.md", "x"), ("skills/b/SKILL.md", "x"),
175+
("rules/r.mdc", "x"), ("snippets/s.py", "x\n" * 76),
176+
("examples/e/README.md", "x"),
177+
("examples/gallery.json", json.dumps({"examples": []})),
178+
("README.md", self.GOOD)):
179+
write(root, rel, text)
180+
(root / "templates" / "t").mkdir(parents=True)
181+
(root / "showcase").mkdir()
182+
e = check_counts.check(root)
183+
self.assertTrue(any("snippets/s.py is 76 lines" in m for m in e), e)
184+
185+
186+
class ExampleRules(unittest.TestCase):
187+
def file_errors(self, src: str) -> list[str]:
188+
with tempfile.TemporaryDirectory() as td:
189+
root = Path(td)
190+
p = write(root, "examples/x/x.py", src)
191+
return cer.check_file(p, root) + cer.check_use_nodes_gated(p, root)
192+
193+
def test_per_vertex_loop_is_flagged(self):
194+
e = self.file_errors("for v in me.vertices:\n v.co = (0, 0, 0)\n")
195+
self.assertTrue(any("foreach_set" in m for m in e), e)
196+
197+
def test_exempt_marker_passes(self):
198+
src = f"for v in me.vertices: {cer.MARKER} twelve verts\n v.co = (0, 0, 0)\n"
199+
self.assertEqual(self.file_errors(src), [])
200+
201+
def test_unguarded_active_object_is_flagged(self):
202+
e = self.file_errors("import bpy\nobj = bpy.context.active_object\nobj.name = 'x'\n")
203+
self.assertTrue(any("is None" in m for m in e), e)
204+
205+
def test_guarded_active_object_passes(self):
206+
src = ("import bpy\nobj = bpy.context.active_object\n"
207+
"if obj is None:\n raise SystemExit(3)\n")
208+
self.assertEqual(self.file_errors(src), [])
209+
210+
def test_ungated_use_nodes_is_flagged(self):
211+
e = self.file_errors("mat.use_nodes = True\n")
212+
self.assertTrue(any("not gated" in m for m in e), e)
213+
214+
def test_gated_use_nodes_passes(self):
215+
src = "import bpy\nif bpy.app.version < (5, 0, 0):\n mat.use_nodes = True\n"
216+
self.assertEqual(self.file_errors(src), [])
217+
218+
219+
class FalsifierTargets(unittest.TestCase):
220+
SCRIPT = ("import argparse\np = argparse.ArgumentParser()\n"
221+
"p.add_argument('--skip-decimate', action='store_true')\n")
222+
EXITS = "| Code | Meaning |\n| --- | --- |\n| 0 | ok |\n| 9 | LOD ratio |\n\n"
223+
224+
def static(self, falsifier_table: str, script: str | None = None):
225+
with tempfile.TemporaryDirectory() as td:
226+
root = Path(td)
227+
readme = write(root, "p/README.md", self.EXITS + falsifier_table)
228+
py = write(root, "p/p.py", self.SCRIPT if script is None else script)
229+
with mock.patch.object(cft, "piece_paths",
230+
lambda: iter([("p", str(readme), str(py))])):
231+
return cft.check_static()[0]
232+
233+
TABLE = "| Flag | Budget violated | Exit |\n| --- | --- | --- |\n| `--skip-decimate` | {t} | {c} |\n"
234+
235+
def test_good_table_passes(self):
236+
self.assertEqual(self.static(self.TABLE.format(t="LOD1 ratio band", c=9)), [])
237+
238+
def test_flag_not_in_argparse(self):
239+
e = self.static(self.TABLE.format(t="LOD1", c=9), script="import argparse\n")
240+
self.assertTrue(any("not an argparse flag" in m for _, m in e), e)
241+
242+
def test_exit_missing_from_exit_table(self):
243+
e = self.static(self.TABLE.format(t="LOD1", c=12))
244+
self.assertTrue(any("absent from the exit-code table" in m for _, m in e), e)
245+
246+
def test_forbidden_exit(self):
247+
e = self.static(self.TABLE.format(t="LOD1", c=2))
248+
self.assertTrue(any("names no check" in m for _, m in e), e)
249+
250+
def test_missing_budget(self):
251+
e = self.static(self.TABLE.format(t="", c=9))
252+
self.assertTrue(any("names no target budget" in m for _, m in e), e)
253+
254+
def test_no_falsifier_table(self):
255+
e = self.static("")
256+
self.assertTrue(any("no falsifier table" in m for _, m in e), e)
257+
258+
259+
if __name__ == "__main__":
260+
unittest.main()

0 commit comments

Comments
 (0)