|
| 1 | +"""The file counts CLAUDE.md and README.md advertise must match the repository. |
| 2 | +
|
| 3 | +`docs/README.md` carries per-folder counts that are correct, and they are correct |
| 4 | +because `test_docs_index_matches_the_tree.py` checks them against a live |
| 5 | +enumeration. The same numbers in `CLAUDE.md` and `README.md` had no such check, |
| 6 | +and every one of them drifted: seven of the nine claims below were wrong when |
| 7 | +this test was written, two of them by more than a factor of two. |
| 8 | +
|
| 9 | +That asymmetry is the whole argument for this file. Correcting the numbers once |
| 10 | +fixes today; the numbers went stale because nothing could tell they had, and |
| 11 | +nothing would have told us again. |
| 12 | +
|
| 13 | +Counts come from `git ls-files`, not from the filesystem, for the reason |
| 14 | +`test_docs_index_matches_the_tree.py` records: the documentation describes the |
| 15 | +repository, not whoever's working tree happens to be on disk. A gitignored |
| 16 | +scratch file would otherwise make the claim wrong in every clone but this one. |
| 17 | +
|
| 18 | +Deliberately NOT checked: `CLAUDE.md`'s "578 classes, 3,985 methods" call-graph |
| 19 | +figures and its "13 servers" MCP count. The first would need a full AST build per |
| 20 | +test run and would churn on any code change; the second describes repositories |
| 21 | +that live elsewhere and cannot be enumerated from here. A check that is expensive |
| 22 | +or that cannot see its subject is worse than an unchecked number, because it |
| 23 | +teaches people to ignore it. |
| 24 | +""" |
| 25 | + |
| 26 | +import io |
| 27 | +import re |
| 28 | +import subprocess |
| 29 | +from pathlib import Path |
| 30 | + |
| 31 | +REPO_ROOT = Path(__file__).resolve().parents[1] |
| 32 | + |
| 33 | + |
| 34 | +def tracked_paths(): |
| 35 | + """List every path tracked in the repository. |
| 36 | +
|
| 37 | + Returns: |
| 38 | + list[str]: Repository-relative paths, forward-slash separated. |
| 39 | + """ |
| 40 | + listing = subprocess.run( |
| 41 | + ["git", "ls-files"], |
| 42 | + cwd=str(REPO_ROOT), |
| 43 | + capture_output=True, |
| 44 | + text=True, |
| 45 | + check=True, |
| 46 | + ) |
| 47 | + return [line for line in listing.stdout.splitlines() if line.strip()] |
| 48 | + |
| 49 | + |
| 50 | +def _python_under(prefix, paths): |
| 51 | + """Count tracked Python files beneath a directory prefix.""" |
| 52 | + return sum(1 for p in paths if p.startswith(prefix) and p.endswith(".py")) |
| 53 | + |
| 54 | + |
| 55 | +def _test_modules_in(prefix, paths): |
| 56 | + """Count tracked ``test_*.py`` modules DIRECTLY inside a directory prefix. |
| 57 | +
|
| 58 | + The prefix ends in a slash, so a file sitting directly in it carries exactly |
| 59 | + as many slashes as the prefix does -- ``tests/`` has one, and so does |
| 60 | + ``tests/test_thing.py``. An earlier version added one to that, which made the |
| 61 | + "unit tests" counter report the nine files one level DEEPER |
| 62 | + (``tests/*/test_*.py``) and the three per-directory counters report zero. |
| 63 | + """ |
| 64 | + depth = prefix.count("/") |
| 65 | + return sum( |
| 66 | + 1 |
| 67 | + for p in paths |
| 68 | + if p.startswith(prefix) |
| 69 | + and p.endswith(".py") |
| 70 | + and p.count("/") == depth |
| 71 | + and p.rsplit("/", 1)[1].startswith("test_") |
| 72 | + ) |
| 73 | + |
| 74 | + |
| 75 | +def actual_counts(): |
| 76 | + """Compute every figure the documentation claims. |
| 77 | +
|
| 78 | + Returns: |
| 79 | + dict: Claim key mapped to the count the repository actually holds. |
| 80 | + """ |
| 81 | + paths = tracked_paths() |
| 82 | + per_dir = {name: _test_modules_in("tests/%s/" % name, paths) for name in ("integration", "e2e", "load")} |
| 83 | + unit = _test_modules_in("tests/", paths) |
| 84 | + return { |
| 85 | + "engine_py": _python_under("langgraph_engine/", paths), |
| 86 | + "repo_py": sum(1 for p in paths if p.endswith(".py")), |
| 87 | + "tests_all_py": _python_under("tests/", paths), |
| 88 | + "tests_total": unit + sum(per_dir.values()), |
| 89 | + "tests_unit": unit, |
| 90 | + "tests_integration": per_dir["integration"], |
| 91 | + "tests_e2e": per_dir["e2e"], |
| 92 | + "tests_load": per_dir["load"], |
| 93 | + "docs_files": sum(1 for p in paths if p.startswith("docs/")), |
| 94 | + } |
| 95 | + |
| 96 | + |
| 97 | +# Each claim names the file it lives in, a pattern whose capture groups are the |
| 98 | +# numbers it advertises, and the actual_counts() keys those groups must equal. |
| 99 | +# The group count and key count must agree -- test_every_claim_is_well_formed |
| 100 | +# asserts that, so a mis-specified claim fails loudly instead of silently |
| 101 | +# checking nothing. |
| 102 | +CLAIMS = ( |
| 103 | + ( |
| 104 | + "CLAUDE.md Quick Info: Total Python Files", |
| 105 | + "CLAUDE.md", |
| 106 | + r"\|\s*\*\*Total Python Files\*\*\s*\|\s*(\d+) \(langgraph_engine/\); (\d+) repo-wide\s*\|", |
| 107 | + ("engine_py", "repo_py"), |
| 108 | + ), |
| 109 | + ( |
| 110 | + "CLAUDE.md Quick Info: Test Files", |
| 111 | + "CLAUDE.md", |
| 112 | + r"\|\s*\*\*Test Files\*\*\s*\|\s*(\d+) \((\d+) unit, (\d+) integration, (\d+) e2e, (\d+) load\)\s*\|", |
| 113 | + ("tests_total", "tests_unit", "tests_integration", "tests_e2e", "tests_load"), |
| 114 | + ), |
| 115 | + ( |
| 116 | + "README.md tests badge", |
| 117 | + "README.md", |
| 118 | + r"Tests-(\d+)%20files%20\xb7%20(\d+)%20integration", |
| 119 | + ("tests_total", "tests_integration"), |
| 120 | + ), |
| 121 | + ( |
| 122 | + "README.md directory tree: tests/ header", |
| 123 | + "README.md", |
| 124 | + r"#\s*(\d+) test_\*\.py files \((\d+) total Python files\)", |
| 125 | + ("tests_total", "tests_all_py"), |
| 126 | + ), |
| 127 | + ( |
| 128 | + "README.md directory tree: unit tests", |
| 129 | + "README.md", |
| 130 | + r"#\s*(\d+) unit tests", |
| 131 | + ("tests_unit",), |
| 132 | + ), |
| 133 | + ( |
| 134 | + "README.md directory tree: integration tests", |
| 135 | + "README.md", |
| 136 | + r"#\s*(\d+) integration tests", |
| 137 | + ("tests_integration",), |
| 138 | + ), |
| 139 | + ( |
| 140 | + "README.md directory tree: e2e tests", |
| 141 | + "README.md", |
| 142 | + r"#\s*(\d+) end-to-end scenario tests", |
| 143 | + ("tests_e2e",), |
| 144 | + ), |
| 145 | + ( |
| 146 | + "README.md directory tree: load tests", |
| 147 | + "README.md", |
| 148 | + r"#\s*(\d+) concurrency / load test", |
| 149 | + ("tests_load",), |
| 150 | + ), |
| 151 | + ( |
| 152 | + "README.md directory tree: docs/", |
| 153 | + "README.md", |
| 154 | + r"#\s*(\d+) files — architecture docs", |
| 155 | + ("docs_files",), |
| 156 | + ), |
| 157 | + ( |
| 158 | + "README.md Testing section: header", |
| 159 | + "README.md", |
| 160 | + r"(\d+) test_\*\.py files \((\d+) total Python files in `tests/`", |
| 161 | + ("tests_total", "tests_all_py"), |
| 162 | + ), |
| 163 | + ( |
| 164 | + "README.md Testing section: Unit tests row", |
| 165 | + "README.md", |
| 166 | + r"\| Unit tests \|\s*(\d+)\s*\|", |
| 167 | + ("tests_unit",), |
| 168 | + ), |
| 169 | + ( |
| 170 | + "README.md Testing section: Integration tests row", |
| 171 | + "README.md", |
| 172 | + r"\| Integration tests \|\s*(\d+)\s*\|", |
| 173 | + ("tests_integration",), |
| 174 | + ), |
| 175 | + ( |
| 176 | + "README.md Testing section: E2E tests row", |
| 177 | + "README.md", |
| 178 | + r"\| E2E tests \|\s*(\d+)\s*\|", |
| 179 | + ("tests_e2e",), |
| 180 | + ), |
| 181 | + ( |
| 182 | + "README.md Testing section: Load tests row", |
| 183 | + "README.md", |
| 184 | + r"\| Load tests \|\s*(\d+)\s*\|", |
| 185 | + ("tests_load",), |
| 186 | + ), |
| 187 | +) |
| 188 | + |
| 189 | + |
| 190 | +def declared(claim): |
| 191 | + """Extract the numbers one claim advertises. |
| 192 | +
|
| 193 | + Args: |
| 194 | + claim: A CLAIMS entry. |
| 195 | +
|
| 196 | + Returns: |
| 197 | + tuple[int, ...] | None: The captured numbers, or None if the pattern |
| 198 | + found no match at all. |
| 199 | + """ |
| 200 | + _label, filename, pattern, _keys = claim |
| 201 | + text = io.open(REPO_ROOT / filename, encoding="utf-8").read() |
| 202 | + found = re.search(pattern, text) |
| 203 | + if found is None: |
| 204 | + return None |
| 205 | + return tuple(int(group) for group in found.groups()) |
| 206 | + |
| 207 | + |
| 208 | +class TestEveryAdvertisedCountIsTrue: |
| 209 | + """A number printed in the documentation must equal the tracked tree.""" |
| 210 | + |
| 211 | + def test_declared_counts_equal_the_tree(self): |
| 212 | + actual = actual_counts() |
| 213 | + wrong = {} |
| 214 | + for claim in CLAIMS: |
| 215 | + label, _filename, _pattern, keys = claim |
| 216 | + numbers = declared(claim) |
| 217 | + assert numbers is not None, "claim pattern matched nothing: %s" % label |
| 218 | + expected = tuple(actual[key] for key in keys) |
| 219 | + if numbers != expected: |
| 220 | + wrong[label] = {"says": numbers, "actual": expected} |
| 221 | + assert wrong == {}, "documentation counts disagree with the tree: %r" % wrong |
| 222 | + |
| 223 | + def test_the_check_can_fail(self): |
| 224 | + """An off-by-one is caught, and both figures are named.""" |
| 225 | + actual = {"tests_total": 99} |
| 226 | + numbers, keys = (98,), ("tests_total",) |
| 227 | + expected = tuple(actual[key] for key in keys) |
| 228 | + assert numbers != expected |
| 229 | + |
| 230 | + |
| 231 | +class TestTheClaimsAreWellFormed: |
| 232 | + """A claim that cannot match, or matches nothing, checks nothing.""" |
| 233 | + |
| 234 | + def test_every_claim_pattern_matches_its_file(self): |
| 235 | + unmatched = [claim[0] for claim in CLAIMS if declared(claim) is None] |
| 236 | + assert unmatched == [], "these claim patterns found no match: %r" % unmatched |
| 237 | + |
| 238 | + def test_group_count_equals_key_count(self): |
| 239 | + """A pattern with fewer groups than keys would silently skip a number.""" |
| 240 | + mismatched = {} |
| 241 | + for label, _filename, pattern, keys in CLAIMS: |
| 242 | + groups = re.compile(pattern).groups |
| 243 | + if groups != len(keys): |
| 244 | + mismatched[label] = (groups, len(keys)) |
| 245 | + assert mismatched == {}, "claim group/key arity disagrees: %r" % mismatched |
| 246 | + |
| 247 | + def test_the_claim_set_is_not_empty(self): |
| 248 | + """Guards against a refactor that leaves the suite checking nothing.""" |
| 249 | + assert len(CLAIMS) >= 10 |
| 250 | + |
| 251 | + |
| 252 | +class TestTheCountersSeeTheRepository: |
| 253 | + """The counters must return real figures, not zero from a bad prefix.""" |
| 254 | + |
| 255 | + def test_no_counter_returns_zero(self): |
| 256 | + zeros = sorted(key for key, value in actual_counts().items() if value == 0) |
| 257 | + assert zeros == [], "these counters found nothing, so their prefix is wrong: %r" % zeros |
| 258 | + |
| 259 | + def test_the_parts_sum_to_the_total(self): |
| 260 | + """tests_total must be the sum of its four parts, or the split is a lie.""" |
| 261 | + actual = actual_counts() |
| 262 | + parts = actual["tests_unit"] + actual["tests_integration"] + actual["tests_e2e"] + actual["tests_load"] |
| 263 | + assert parts == actual["tests_total"] |
0 commit comments