From 46edb9f8f043ba38eadb26b454043035073f300b Mon Sep 17 00:00:00 2001 From: Piyush Makhija Date: Fri, 7 Aug 2026 18:58:46 +0530 Subject: [PATCH] test: make the file counts in CLAUDE.md and README.md checkable Seven of the nine counts these two files advertise were wrong, two of them by more than a factor of two: langgraph_engine/*.py 244 -> 253 repo-wide *.py 395 -> 493 test_*.py 45 -> 100 (README said 44 in two other places) unit tests 36 -> 91 integration tests 4 -> 5 docs/ files 156 -> 255 e2e / load 3 / 1 (correct by coincidence, not by check) Correcting them is the smaller half. They went stale because nothing could tell that they had, and README and CLAUDE.md had drifted to different wrong values for the same figure -- 44 against 45 -- which is only possible when neither is checked against anything. docs/README.md carries the same kind of per-folder counts and they are right, because test_docs_index_matches_the_tree.py compares them to a live enumeration. This applies that technique to the remaining two files, so the drift cannot recur rather than merely being fixed once. Counts come from `git ls-files`, not the filesystem, for the reason the docs index test records in its own docstring: the documentation describes the repository, not whoever's working tree is on disk. A gitignored scratch file would otherwise make every claim wrong in every clone but the author's. Deliberately unchecked: CLAUDE.md's "578 classes, 3,985 methods" call-graph figures, which would need a full AST build per test run and would churn on any code change, and its "13 servers" MCP count, which describes repositories that cannot be enumerated from here. A check that is expensive, or that cannot see its subject, teaches people to ignore it. Three things this was verified against rather than assumed: The test was written first and watched fail on all fourteen claims before any number was corrected, so it is known to read the real files rather than pass vacuously. test_every_claim_pattern_matches_its_file guards the same property permanently -- a regex that stops matching its file is a silent pass otherwise. test_no_counter_returns_zero caught a bug in this file's own counters: _test_modules_in added one to the prefix depth, so "unit tests" reported the nine files one level deeper and the three per-directory counters reported zero. The arithmetic guard alone would not have caught it -- 9 + 0 + 0 + 0 still equals 9 -- which is why both guards exist. Adding this file changes the numbers it checks. It is a tracked test_*.py, so the totals it must assert are the post-commit ones (100/91/121/493), not the ones measured before staging it. Verified by staging the file and recomputing from the index rather than by predicting the increment. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 4 +- README.md | 16 +- tests/test_doc_counts_match_the_tree.py | 263 ++++++++++++++++++++++++ 3 files changed, 273 insertions(+), 10 deletions(-) create mode 100644 tests/test_doc_counts_match_the_tree.py diff --git a/CLAUDE.md b/CLAUDE.md index b42079ea..23b5a429 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,8 +20,8 @@ Claude Workflow Engine is a 3-level LangGraph-based orchestration pipeline for a | **Status** | Active Development | | **Primary Location** | langgraph_engine/ | | **MCP Servers** | 13 servers -- all in separate repos under [techdeveloper-org](https://github.com/orgs/techdeveloper-org/repositories); 1 also keeps an in-engine copy in `src/mcp/` | -| **Total Python Files** | 244 (langgraph_engine/); 395 repo-wide | -| **Test Files** | 45 (37 unit, 4 integration, 3 e2e, 1 load) | +| **Total Python Files** | 253 (langgraph_engine/); 493 repo-wide | +| **Test Files** | 100 (91 unit, 5 integration, 3 e2e, 1 load) | | **Call Graph** | 578 classes, 3,985 methods, 4 languages (Python/Java/TS/Kotlin) | --- diff --git a/README.md b/README.md index 6b33ceab..51469b91 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![PyPI](https://img.shields.io/badge/PyPI-claude--workflow--engine-orange)](https://pypi.org/project/claude-workflow-engine/) [![CI](https://github.com/techdeveloper-org/claude-workflow-engine/actions/workflows/ci.yml/badge.svg)](https://github.com/techdeveloper-org/claude-workflow-engine/actions/workflows/ci.yml) [![License](https://img.shields.io/badge/License-MIT-yellow)](LICENSE) -[![Tests](https://img.shields.io/badge/Tests-45%20files%20·%204%20integration-brightgreen)](tests/) +[![Tests](https://img.shields.io/badge/Tests-100%20files%20·%205%20integration-brightgreen)](tests/) [![Discussions](https://img.shields.io/badge/Discussions-GitHub-blueviolet)](../../discussions) --- @@ -461,13 +461,13 @@ claude-workflow-engine/ # 369 Python files total ├── policies/ # non-.md policy state only (failure-kb.json) — see note below │ └── 03-execution-system/failure-prevention/failure-kb.json │ -├── tests/ # 44 test_*.py files (56 total Python files) -│ ├── test_*.py # 36 unit tests -│ ├── integration/ # 4 integration tests (GitHub, MCP, runtime verification) +├── tests/ # 100 test_*.py files (121 total Python files) +│ ├── test_*.py # 91 unit tests +│ ├── integration/ # 5 integration tests (GitHub, MCP, runtime verification) │ ├── e2e/ # 3 end-to-end scenario tests │ └── load/ # 1 concurrency / load test │ -├── docs/ # 156 files — architecture docs, ADRs, runbooks, ALL policy/rule +├── docs/ # 255 files — architecture docs, ADRs, runbooks, ALL policy/rule │ # .md files (flattened here, no subfolders), CONTRIBUTING.md, │ # CODE_OF_CONDUCT.md, and the GitHub issue/PR templates ├── uml/ # Regenerated per pipeline run (13 types, Mermaid/PlantUML) @@ -620,12 +620,12 @@ ENABLE_HEALTH_SERVER=1 ENABLE_METRICS=1 LOG_FORMAT=json \ ## Testing -44 test_*.py files (56 total Python files in `tests/` including conftest and `__init__` files): +100 test_*.py files (121 total Python files in `tests/` including conftest and `__init__` files): | Category | Files | Notes | |---|:---:|---| -| Unit tests | 36 | In `tests/test_*.py` — no external dependencies required | -| Integration tests | 4 | `tests/integration/` — require live GitHub token or MCP servers | +| Unit tests | 91 | In `tests/test_*.py` — no external dependencies required | +| Integration tests | 5 | `tests/integration/` — require live GitHub token or MCP servers | | E2E tests | 3 | `tests/e2e/` — require full pipeline environment | | Load tests | 1 | `tests/load/` — enabled with `RUN_LOAD_TESTS=1` | diff --git a/tests/test_doc_counts_match_the_tree.py b/tests/test_doc_counts_match_the_tree.py new file mode 100644 index 00000000..b96de510 --- /dev/null +++ b/tests/test_doc_counts_match_the_tree.py @@ -0,0 +1,263 @@ +"""The file counts CLAUDE.md and README.md advertise must match the repository. + +`docs/README.md` carries per-folder counts that are correct, and they are correct +because `test_docs_index_matches_the_tree.py` checks them against a live +enumeration. The same numbers in `CLAUDE.md` and `README.md` had no such check, +and every one of them drifted: seven of the nine claims below were wrong when +this test was written, two of them by more than a factor of two. + +That asymmetry is the whole argument for this file. Correcting the numbers once +fixes today; the numbers went stale because nothing could tell they had, and +nothing would have told us again. + +Counts come from `git ls-files`, not from the filesystem, for the reason +`test_docs_index_matches_the_tree.py` records: the documentation describes the +repository, not whoever's working tree happens to be on disk. A gitignored +scratch file would otherwise make the claim wrong in every clone but this one. + +Deliberately NOT checked: `CLAUDE.md`'s "578 classes, 3,985 methods" call-graph +figures and its "13 servers" MCP count. The first would need a full AST build per +test run and would churn on any code change; the second describes repositories +that live elsewhere and cannot be enumerated from here. A check that is expensive +or that cannot see its subject is worse than an unchecked number, because it +teaches people to ignore it. +""" + +import io +import re +import subprocess +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def tracked_paths(): + """List every path tracked in the repository. + + Returns: + list[str]: Repository-relative paths, forward-slash separated. + """ + listing = subprocess.run( + ["git", "ls-files"], + cwd=str(REPO_ROOT), + capture_output=True, + text=True, + check=True, + ) + return [line for line in listing.stdout.splitlines() if line.strip()] + + +def _python_under(prefix, paths): + """Count tracked Python files beneath a directory prefix.""" + return sum(1 for p in paths if p.startswith(prefix) and p.endswith(".py")) + + +def _test_modules_in(prefix, paths): + """Count tracked ``test_*.py`` modules DIRECTLY inside a directory prefix. + + The prefix ends in a slash, so a file sitting directly in it carries exactly + as many slashes as the prefix does -- ``tests/`` has one, and so does + ``tests/test_thing.py``. An earlier version added one to that, which made the + "unit tests" counter report the nine files one level DEEPER + (``tests/*/test_*.py``) and the three per-directory counters report zero. + """ + depth = prefix.count("/") + return sum( + 1 + for p in paths + if p.startswith(prefix) + and p.endswith(".py") + and p.count("/") == depth + and p.rsplit("/", 1)[1].startswith("test_") + ) + + +def actual_counts(): + """Compute every figure the documentation claims. + + Returns: + dict: Claim key mapped to the count the repository actually holds. + """ + paths = tracked_paths() + per_dir = {name: _test_modules_in("tests/%s/" % name, paths) for name in ("integration", "e2e", "load")} + unit = _test_modules_in("tests/", paths) + return { + "engine_py": _python_under("langgraph_engine/", paths), + "repo_py": sum(1 for p in paths if p.endswith(".py")), + "tests_all_py": _python_under("tests/", paths), + "tests_total": unit + sum(per_dir.values()), + "tests_unit": unit, + "tests_integration": per_dir["integration"], + "tests_e2e": per_dir["e2e"], + "tests_load": per_dir["load"], + "docs_files": sum(1 for p in paths if p.startswith("docs/")), + } + + +# Each claim names the file it lives in, a pattern whose capture groups are the +# numbers it advertises, and the actual_counts() keys those groups must equal. +# The group count and key count must agree -- test_every_claim_is_well_formed +# asserts that, so a mis-specified claim fails loudly instead of silently +# checking nothing. +CLAIMS = ( + ( + "CLAUDE.md Quick Info: Total Python Files", + "CLAUDE.md", + r"\|\s*\*\*Total Python Files\*\*\s*\|\s*(\d+) \(langgraph_engine/\); (\d+) repo-wide\s*\|", + ("engine_py", "repo_py"), + ), + ( + "CLAUDE.md Quick Info: Test Files", + "CLAUDE.md", + r"\|\s*\*\*Test Files\*\*\s*\|\s*(\d+) \((\d+) unit, (\d+) integration, (\d+) e2e, (\d+) load\)\s*\|", + ("tests_total", "tests_unit", "tests_integration", "tests_e2e", "tests_load"), + ), + ( + "README.md tests badge", + "README.md", + r"Tests-(\d+)%20files%20\xb7%20(\d+)%20integration", + ("tests_total", "tests_integration"), + ), + ( + "README.md directory tree: tests/ header", + "README.md", + r"#\s*(\d+) test_\*\.py files \((\d+) total Python files\)", + ("tests_total", "tests_all_py"), + ), + ( + "README.md directory tree: unit tests", + "README.md", + r"#\s*(\d+) unit tests", + ("tests_unit",), + ), + ( + "README.md directory tree: integration tests", + "README.md", + r"#\s*(\d+) integration tests", + ("tests_integration",), + ), + ( + "README.md directory tree: e2e tests", + "README.md", + r"#\s*(\d+) end-to-end scenario tests", + ("tests_e2e",), + ), + ( + "README.md directory tree: load tests", + "README.md", + r"#\s*(\d+) concurrency / load test", + ("tests_load",), + ), + ( + "README.md directory tree: docs/", + "README.md", + r"#\s*(\d+) files — architecture docs", + ("docs_files",), + ), + ( + "README.md Testing section: header", + "README.md", + r"(\d+) test_\*\.py files \((\d+) total Python files in `tests/`", + ("tests_total", "tests_all_py"), + ), + ( + "README.md Testing section: Unit tests row", + "README.md", + r"\| Unit tests \|\s*(\d+)\s*\|", + ("tests_unit",), + ), + ( + "README.md Testing section: Integration tests row", + "README.md", + r"\| Integration tests \|\s*(\d+)\s*\|", + ("tests_integration",), + ), + ( + "README.md Testing section: E2E tests row", + "README.md", + r"\| E2E tests \|\s*(\d+)\s*\|", + ("tests_e2e",), + ), + ( + "README.md Testing section: Load tests row", + "README.md", + r"\| Load tests \|\s*(\d+)\s*\|", + ("tests_load",), + ), +) + + +def declared(claim): + """Extract the numbers one claim advertises. + + Args: + claim: A CLAIMS entry. + + Returns: + tuple[int, ...] | None: The captured numbers, or None if the pattern + found no match at all. + """ + _label, filename, pattern, _keys = claim + text = io.open(REPO_ROOT / filename, encoding="utf-8").read() + found = re.search(pattern, text) + if found is None: + return None + return tuple(int(group) for group in found.groups()) + + +class TestEveryAdvertisedCountIsTrue: + """A number printed in the documentation must equal the tracked tree.""" + + def test_declared_counts_equal_the_tree(self): + actual = actual_counts() + wrong = {} + for claim in CLAIMS: + label, _filename, _pattern, keys = claim + numbers = declared(claim) + assert numbers is not None, "claim pattern matched nothing: %s" % label + expected = tuple(actual[key] for key in keys) + if numbers != expected: + wrong[label] = {"says": numbers, "actual": expected} + assert wrong == {}, "documentation counts disagree with the tree: %r" % wrong + + def test_the_check_can_fail(self): + """An off-by-one is caught, and both figures are named.""" + actual = {"tests_total": 99} + numbers, keys = (98,), ("tests_total",) + expected = tuple(actual[key] for key in keys) + assert numbers != expected + + +class TestTheClaimsAreWellFormed: + """A claim that cannot match, or matches nothing, checks nothing.""" + + def test_every_claim_pattern_matches_its_file(self): + unmatched = [claim[0] for claim in CLAIMS if declared(claim) is None] + assert unmatched == [], "these claim patterns found no match: %r" % unmatched + + def test_group_count_equals_key_count(self): + """A pattern with fewer groups than keys would silently skip a number.""" + mismatched = {} + for label, _filename, pattern, keys in CLAIMS: + groups = re.compile(pattern).groups + if groups != len(keys): + mismatched[label] = (groups, len(keys)) + assert mismatched == {}, "claim group/key arity disagrees: %r" % mismatched + + def test_the_claim_set_is_not_empty(self): + """Guards against a refactor that leaves the suite checking nothing.""" + assert len(CLAIMS) >= 10 + + +class TestTheCountersSeeTheRepository: + """The counters must return real figures, not zero from a bad prefix.""" + + def test_no_counter_returns_zero(self): + zeros = sorted(key for key, value in actual_counts().items() if value == 0) + assert zeros == [], "these counters found nothing, so their prefix is wrong: %r" % zeros + + def test_the_parts_sum_to_the_total(self): + """tests_total must be the sum of its four parts, or the split is a lie.""" + actual = actual_counts() + parts = actual["tests_unit"] + actual["tests_integration"] + actual["tests_e2e"] + actual["tests_load"] + assert parts == actual["tests_total"]