|
| 1 | +#!/usr/bin/env python3 |
| 2 | +""" |
| 3 | +lint-memory-core.py — the memory-core linter. |
| 4 | +
|
| 5 | +Memory in this project is allocated through ONE core (src/foundation/mem_core.h), |
| 6 | +not through scattered raw malloc/calloc/realloc/free/strdup. This linter is the |
| 7 | +gate that keeps that true. |
| 8 | +
|
| 9 | +WHY A RATCHET, NOT A BAN. An audit on 2026-09-13 found ~800 raw allocation |
| 10 | +sites in src/ and none of them can be migrated in one change. So the gate works |
| 11 | +like every other honest debt gate: a checked-in baseline records how many raw |
| 12 | +sites each file has TODAY, and the build goes red the moment any file has MORE |
| 13 | +than its baseline, or a file not in the baseline grows one. Files can only ever |
| 14 | +go down. When a file is migrated, its baseline line is lowered (or removed) in |
| 15 | +the same change -- and --strict turns "below baseline" into a failure too, so |
| 16 | +the baseline cannot silently rot behind the code. |
| 17 | +
|
| 18 | +WHAT COUNTS AS RAW. A call to malloc/calloc/realloc/free/strdup/strndup that is |
| 19 | +not part of a longer identifier. cbm_alloc, cbm_free, cbm_calloc, mi_malloc, |
| 20 | +cbm_arena_alloc and heap_strdup are all NOT matches: the character before the |
| 21 | +name is an identifier character. Comments and string literals are stripped |
| 22 | +first, so prose that mentions malloc( does not trip the gate -- the security |
| 23 | +audit already bit us once on exactly that with fork(. |
| 24 | +
|
| 25 | +EXEMPT. The core itself and the allocator plumbing it sits on: |
| 26 | + src/foundation/mem_core.c the route |
| 27 | + src/foundation/arena.c bulk allocator that reports INTO the core |
| 28 | + src/foundation/slab_alloc.c same |
| 29 | + src/foundation/mem.c policy/measurement, probes with malloc |
| 30 | + src/foundation/mem_override_*.c the --wrap / override shims |
| 31 | + src/foundation/compat*.c libc replacement surface |
| 32 | + internal/cbm/vendored/** not ours |
| 33 | + vendored/** not ours |
| 34 | +
|
| 35 | +Usage: |
| 36 | + lint-memory-core.py check against the baseline (CI) |
| 37 | + lint-memory-core.py --strict also fail when a file is BELOW baseline |
| 38 | + lint-memory-core.py --write-baseline regenerate scripts/memory-core-baseline.txt |
| 39 | + lint-memory-core.py --list print every raw site (file:line: call) |
| 40 | +
|
| 41 | +Exit 0 = clean. Exit 1 = a file grew. Exit 2 = usage/IO error. |
| 42 | +""" |
| 43 | +from __future__ import annotations |
| 44 | + |
| 45 | +import argparse |
| 46 | +import pathlib |
| 47 | +import re |
| 48 | +import sys |
| 49 | + |
| 50 | +ROOT = pathlib.Path(__file__).resolve().parent.parent |
| 51 | +BASELINE = ROOT / "scripts" / "memory-core-baseline.txt" |
| 52 | +SCAN_ROOTS = ("src", "internal/cbm") |
| 53 | +RAW = re.compile(r"(?<![A-Za-z0-9_])(malloc|calloc|realloc|free|strdup|strndup)\s*\(") |
| 54 | + |
| 55 | +EXEMPT_EXACT = { |
| 56 | + "src/foundation/mem_core.c", |
| 57 | + "src/foundation/arena.c", |
| 58 | + "src/foundation/slab_alloc.c", |
| 59 | + "src/foundation/mem.c", |
| 60 | +} |
| 61 | +EXEMPT_PREFIX = ( |
| 62 | + "src/foundation/mem_override_", |
| 63 | + "src/foundation/compat", |
| 64 | + "internal/cbm/vendored/", |
| 65 | + "vendored/", |
| 66 | +) |
| 67 | + |
| 68 | + |
| 69 | +def is_exempt(rel: str) -> bool: |
| 70 | + return rel in EXEMPT_EXACT or rel.startswith(EXEMPT_PREFIX) |
| 71 | + |
| 72 | + |
| 73 | +def strip_comments_and_strings(text: str) -> str: |
| 74 | + """Blank out comments and string literals, preserving line structure so |
| 75 | + reported line numbers stay right. Character-class aware enough for C: |
| 76 | + handles escapes inside strings and does not treat // inside a string as a |
| 77 | + comment.""" |
| 78 | + out = [] |
| 79 | + i, n = 0, len(text) |
| 80 | + while i < n: |
| 81 | + c = text[i] |
| 82 | + nxt = text[i + 1] if i + 1 < n else "" |
| 83 | + if c == "/" and nxt == "*": |
| 84 | + j = text.find("*/", i + 2) |
| 85 | + j = n if j < 0 else j + 2 |
| 86 | + out.append("".join(ch if ch == "\n" else " " for ch in text[i:j])) |
| 87 | + i = j |
| 88 | + elif c == "/" and nxt == "/": |
| 89 | + j = text.find("\n", i) |
| 90 | + j = n if j < 0 else j |
| 91 | + out.append(" " * (j - i)) |
| 92 | + i = j |
| 93 | + elif c == '"' or c == "'": |
| 94 | + q = c |
| 95 | + j = i + 1 |
| 96 | + while j < n and text[j] != q: |
| 97 | + if text[j] == "\\": |
| 98 | + j += 1 |
| 99 | + if j < n and text[j] == "\n": |
| 100 | + break |
| 101 | + j += 1 |
| 102 | + j = min(j + 1, n) |
| 103 | + out.append(q + " " * max(0, j - i - 2) + (q if j - i >= 2 else "")) |
| 104 | + i = j |
| 105 | + else: |
| 106 | + out.append(c) |
| 107 | + i += 1 |
| 108 | + return "".join(out) |
| 109 | + |
| 110 | + |
| 111 | +def scan() -> dict[str, list[tuple[int, str]]]: |
| 112 | + hits: dict[str, list[tuple[int, str]]] = {} |
| 113 | + for root in SCAN_ROOTS: |
| 114 | + base = ROOT / root |
| 115 | + if not base.exists(): |
| 116 | + continue |
| 117 | + for path in sorted(base.rglob("*")): |
| 118 | + if path.suffix not in (".c", ".h") or not path.is_file(): |
| 119 | + continue |
| 120 | + rel = path.relative_to(ROOT).as_posix() |
| 121 | + if is_exempt(rel): |
| 122 | + continue |
| 123 | + text = strip_comments_and_strings(path.read_text(encoding="utf-8", errors="replace")) |
| 124 | + for lineno, line in enumerate(text.splitlines(), 1): |
| 125 | + for m in RAW.finditer(line): |
| 126 | + hits.setdefault(rel, []).append((lineno, m.group(1))) |
| 127 | + return hits |
| 128 | + |
| 129 | + |
| 130 | +def read_baseline() -> dict[str, int]: |
| 131 | + if not BASELINE.exists(): |
| 132 | + return {} |
| 133 | + out: dict[str, int] = {} |
| 134 | + for raw in BASELINE.read_text(encoding="utf-8").splitlines(): |
| 135 | + line = raw.strip() |
| 136 | + if not line or line.startswith("#"): |
| 137 | + continue |
| 138 | + rel, _, count = line.rpartition("\t") |
| 139 | + if not rel: |
| 140 | + continue |
| 141 | + try: |
| 142 | + out[rel] = int(count) |
| 143 | + except ValueError: |
| 144 | + continue |
| 145 | + return out |
| 146 | + |
| 147 | + |
| 148 | +def write_baseline(hits: dict[str, list]) -> None: |
| 149 | + lines = [ |
| 150 | + "# memory-core-baseline.txt -- raw allocator call sites per file.", |
| 151 | + "# Generated by scripts/lint-memory-core.py --write-baseline.", |
| 152 | + "# A file may only ever go DOWN. Lower a line in the same change that", |
| 153 | + "# migrates the file to src/foundation/mem_core.h; never raise one.", |
| 154 | + "", |
| 155 | + ] |
| 156 | + for rel in sorted(hits): |
| 157 | + lines.append(f"{rel}\t{len(hits[rel])}") |
| 158 | + BASELINE.write_text("\n".join(lines) + "\n", encoding="utf-8") |
| 159 | + |
| 160 | + |
| 161 | +def main() -> int: |
| 162 | + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) |
| 163 | + ap.add_argument("--write-baseline", action="store_true") |
| 164 | + ap.add_argument("--strict", action="store_true", help="fail when a file is BELOW its baseline") |
| 165 | + ap.add_argument("--list", action="store_true", help="print every raw site") |
| 166 | + args = ap.parse_args() |
| 167 | + |
| 168 | + hits = scan() |
| 169 | + total = sum(len(v) for v in hits.values()) |
| 170 | + |
| 171 | + if args.write_baseline: |
| 172 | + write_baseline(hits) |
| 173 | + print(f"memory-core baseline written: {len(hits)} files, {total} raw sites") |
| 174 | + return 0 |
| 175 | + |
| 176 | + if args.list: |
| 177 | + for rel in sorted(hits): |
| 178 | + for lineno, call in hits[rel]: |
| 179 | + print(f"{rel}:{lineno}: {call}(") |
| 180 | + print(f"-- {total} raw sites in {len(hits)} files") |
| 181 | + return 0 |
| 182 | + |
| 183 | + baseline = read_baseline() |
| 184 | + if not baseline: |
| 185 | + print(f"ERROR: no baseline at {BASELINE.relative_to(ROOT)}; run --write-baseline", file=sys.stderr) |
| 186 | + return 2 |
| 187 | + |
| 188 | + grew: list[str] = [] |
| 189 | + shrank: list[str] = [] |
| 190 | + for rel, sites in sorted(hits.items()): |
| 191 | + now = len(sites) |
| 192 | + allowed = baseline.get(rel, 0) |
| 193 | + if now > allowed: |
| 194 | + # Show the LAST sites: new code is usually appended, so these are the |
| 195 | + # likeliest culprits. The linter cannot know which sites are new |
| 196 | + # without a diff; --list prints them all. |
| 197 | + delta = now - allowed |
| 198 | + tail = ", ".join(f"{ln}:{c}" for ln, c in sites[-max(delta, 1):][-6:]) |
| 199 | + grew.append(f" {rel}: grew by {delta} ({allowed} -> {now}); latest sites: {tail}" |
| 200 | + f" [run --list for all]") |
| 201 | + elif now < allowed: |
| 202 | + shrank.append(f" {rel}: {now} (baseline {allowed}) -- lower the baseline") |
| 203 | + for rel, allowed in sorted(baseline.items()): |
| 204 | + if rel not in hits and allowed > 0: |
| 205 | + shrank.append(f" {rel}: 0 (baseline {allowed}) -- remove the line") |
| 206 | + |
| 207 | + if grew: |
| 208 | + print("memory-core linter FAILED: raw allocator use grew. Allocate through") |
| 209 | + print("src/foundation/mem_core.h (cbm_alloc/cbm_calloc/cbm_realloc/cbm_free/") |
| 210 | + print("cbm_mem_strdup) instead of malloc/calloc/realloc/free/strdup.") |
| 211 | + print("\n".join(grew)) |
| 212 | + if shrank: |
| 213 | + print("memory-core ratchet: these files improved; tighten scripts/memory-core-baseline.txt:") |
| 214 | + print("\n".join(shrank)) |
| 215 | + if grew or (args.strict and shrank): |
| 216 | + return 1 |
| 217 | + print(f"memory-core linter passed: {total} raw sites across {len(hits)} files, none grew") |
| 218 | + return 0 |
| 219 | + |
| 220 | + |
| 221 | +if __name__ == "__main__": |
| 222 | + sys.exit(main()) |
0 commit comments