|
| 1 | +"""ctags metadata enrichment for symbol records (roadmap milestone 2). |
| 2 | +
|
| 3 | +Fills the ``kind`` / ``typeref`` / ``scope`` / ``signature`` record fields by |
| 4 | +running Universal Ctags directly on the files that appear in definition-shaped |
| 5 | +results — so ``symbol_info`` can say *what* a symbol is (function, macro, |
| 6 | +struct, typedef, enum constant, ...) with no build and no compile database. |
| 7 | +
|
| 8 | +Everything here is best-effort by design: when Universal Ctags with JSON |
| 9 | +output is not available, or a file cannot be parsed, records simply keep the |
| 10 | +``null`` metadata they had before v0.8.1. No tool ever fails, slows down |
| 11 | +noticeably, or changes shape because of enrichment. |
| 12 | +
|
| 13 | +Cost is bounded: at most one ctags run per distinct file per result page, |
| 14 | +cached in a small LRU keyed by (mtime, size) — including negative results, so |
| 15 | +a file ctags cannot handle is not re-parsed on every query. |
| 16 | +""" |
| 17 | + |
| 18 | +from __future__ import annotations |
| 19 | + |
| 20 | +import json |
| 21 | +import re |
| 22 | +import subprocess |
| 23 | +import threading |
| 24 | +from collections import OrderedDict |
| 25 | +from pathlib import Path |
| 26 | + |
| 27 | +from . import toolchain |
| 28 | + |
| 29 | +CTAGS_TIMEOUT_SECONDS = 10 |
| 30 | +# Belt and braces on top of the timeout: never feed ctags a generated blob. |
| 31 | +MAX_FILE_BYTES = 8 * 1024 * 1024 |
| 32 | +CACHE_CAPACITY = 256 |
| 33 | + |
| 34 | +# ``--fields=*`` asks for everything ctags knows (we pick what we need, and |
| 35 | +# stay compatible if fields are reordered); ``+px`` adds prototypes/externs so |
| 36 | +# header declarations — which gtags may report as definitions — still match. |
| 37 | +CTAGS_ARGS = ( |
| 38 | + "--output-format=json", |
| 39 | + "--fields=*", |
| 40 | + "--kinds-C=+px", |
| 41 | + "--kinds-C++=+px", |
| 42 | + "-o", |
| 43 | + "-", |
| 44 | +) |
| 45 | + |
| 46 | +# When one name+line yields several tags (e.g. ``typedef struct foo foo;`` |
| 47 | +# emits both a struct and a typedef tag), prefer the kind an agent most |
| 48 | +# likely asked about. Unlisted kinds rank last, ties break on list order. |
| 49 | +_KIND_PRIORITY = ( |
| 50 | + "function", |
| 51 | + "typedef", |
| 52 | + "struct", |
| 53 | + "union", |
| 54 | + "enum", |
| 55 | + "class", |
| 56 | + "macro", |
| 57 | + "enumerator", |
| 58 | + "member", |
| 59 | + "variable", |
| 60 | + "prototype", |
| 61 | + "externvar", |
| 62 | +) |
| 63 | + |
| 64 | +# ctags invents names like "__anon0416201d0103" for anonymous enums/structs; |
| 65 | +# hash noise helps nobody — render them as "<anonymous>" instead. |
| 66 | +_ANON_RE = re.compile(r"__anon[0-9a-fA-F]+") |
| 67 | + |
| 68 | +# Capability probe results keyed by resolved ctags path: (usable, detail). |
| 69 | +_probe_cache: dict[str, tuple[bool, str]] = {} |
| 70 | +# LRU: absolute path -> ((mtime_ns, size), name -> [normalized tags]). |
| 71 | +_tags_cache: OrderedDict[str, tuple[tuple[int, int], dict[str, list[dict]]]] = ( |
| 72 | + OrderedDict() |
| 73 | +) |
| 74 | +_lock = threading.Lock() |
| 75 | + |
| 76 | + |
| 77 | +def reset_cache() -> None: |
| 78 | + """Forget probe results and cached tags (used by tests).""" |
| 79 | + with _lock: |
| 80 | + _probe_cache.clear() |
| 81 | + _tags_cache.clear() |
| 82 | + |
| 83 | + |
| 84 | +def _probe(exe: str) -> tuple[bool, str]: |
| 85 | + """Check one ctags binary for Universal Ctags with JSON output support.""" |
| 86 | + with _lock: |
| 87 | + if exe in _probe_cache: |
| 88 | + return _probe_cache[exe] |
| 89 | + try: |
| 90 | + proc = subprocess.run( |
| 91 | + [exe, "--version"], |
| 92 | + capture_output=True, |
| 93 | + text=True, |
| 94 | + errors="replace", |
| 95 | + timeout=CTAGS_TIMEOUT_SECONDS, |
| 96 | + ) |
| 97 | + out = proc.stdout |
| 98 | + except (OSError, subprocess.SubprocessError): |
| 99 | + out = "" |
| 100 | + first = out.splitlines()[0].split(",")[0].strip() if out.strip() else exe |
| 101 | + if not out.strip(): |
| 102 | + result = (False, f"{exe} did not run") |
| 103 | + elif "Universal Ctags" not in out: |
| 104 | + result = (False, f"{first} has no JSON output (Universal Ctags needed)") |
| 105 | + elif "+json" not in out: |
| 106 | + result = (False, f"{first} was built without +json support") |
| 107 | + else: |
| 108 | + result = (True, first) |
| 109 | + with _lock: |
| 110 | + _probe_cache[exe] = result |
| 111 | + return result |
| 112 | + |
| 113 | + |
| 114 | +def available(bin_dir: str | None = None) -> bool: |
| 115 | + """True when a Universal Ctags with JSON output can be invoked.""" |
| 116 | + exe = toolchain.find_ctags(bin_dir) |
| 117 | + return _probe(exe)[0] if exe else False |
| 118 | + |
| 119 | + |
| 120 | +def status_line(bin_dir: str | None = None) -> str: |
| 121 | + """One human-readable line for the ``doctor`` subcommand.""" |
| 122 | + exe = toolchain.find_ctags(bin_dir) |
| 123 | + if exe is None: |
| 124 | + return "metadata enrichment: unavailable (no ctags binary found)" |
| 125 | + usable, detail = _probe(exe) |
| 126 | + if usable: |
| 127 | + return f"metadata enrichment: active ({detail})" |
| 128 | + return f"metadata enrichment: unavailable ({detail})" |
| 129 | + |
| 130 | + |
| 131 | +def _normalize_tag(obj: dict) -> dict | None: |
| 132 | + """One raw ctags JSON object -> {line, kind, typeref, scope, signature}.""" |
| 133 | + line = obj.get("line") |
| 134 | + if obj.get("_type") != "tag" or not obj.get("name") or not isinstance(line, int): |
| 135 | + return None |
| 136 | + typeref = obj.get("typeref") |
| 137 | + if isinstance(typeref, str): |
| 138 | + if typeref.startswith("typename:"): |
| 139 | + # "typename:" is ctags noise; real refs like "struct:item" stay. |
| 140 | + typeref = typeref[len("typename:") :] |
| 141 | + typeref = _ANON_RE.sub("<anonymous>", typeref) |
| 142 | + scope = obj.get("scope") |
| 143 | + if isinstance(scope, str): |
| 144 | + if isinstance(obj.get("scopeKind"), str): |
| 145 | + scope = f"{obj['scopeKind']}:{scope}" |
| 146 | + scope = _ANON_RE.sub("<anonymous>", scope) |
| 147 | + return { |
| 148 | + "line": line, |
| 149 | + "kind": obj.get("kind"), |
| 150 | + "typeref": typeref if isinstance(typeref, str) else None, |
| 151 | + "scope": scope if isinstance(scope, str) else None, |
| 152 | + "signature": obj.get("signature") if isinstance(obj.get("signature"), str) else None, |
| 153 | + } |
| 154 | + |
| 155 | + |
| 156 | +def _run_ctags(exe: str, abs_path: Path) -> dict[str, list[dict]]: |
| 157 | + """Run ctags on one file and index its tags by name. {} on any failure.""" |
| 158 | + try: |
| 159 | + proc = subprocess.run( |
| 160 | + [exe, *CTAGS_ARGS, str(abs_path)], |
| 161 | + capture_output=True, |
| 162 | + text=True, |
| 163 | + errors="replace", |
| 164 | + timeout=CTAGS_TIMEOUT_SECONDS, |
| 165 | + ) |
| 166 | + except (OSError, subprocess.SubprocessError): |
| 167 | + return {} |
| 168 | + if proc.returncode != 0: |
| 169 | + return {} |
| 170 | + tags: dict[str, list[dict]] = {} |
| 171 | + for raw in proc.stdout.splitlines(): |
| 172 | + try: |
| 173 | + obj = json.loads(raw) |
| 174 | + except ValueError: |
| 175 | + continue |
| 176 | + if isinstance(obj, dict) and (tag := _normalize_tag(obj)): |
| 177 | + tags.setdefault(obj["name"], []).append(tag) |
| 178 | + return tags |
| 179 | + |
| 180 | + |
| 181 | +def tags_for_file(abs_path: Path, bin_dir: str | None = None) -> dict[str, list[dict]]: |
| 182 | + """All tags ctags finds in one file, by symbol name. Cached; never raises.""" |
| 183 | + try: |
| 184 | + stat = abs_path.stat() |
| 185 | + except OSError: |
| 186 | + return {} |
| 187 | + signature = (stat.st_mtime_ns, stat.st_size) |
| 188 | + key = str(abs_path) |
| 189 | + with _lock: |
| 190 | + cached = _tags_cache.get(key) |
| 191 | + if cached and cached[0] == signature: |
| 192 | + _tags_cache.move_to_end(key) |
| 193 | + return cached[1] |
| 194 | + if stat.st_size > MAX_FILE_BYTES: |
| 195 | + tags: dict[str, list[dict]] = {} |
| 196 | + else: |
| 197 | + exe = toolchain.find_ctags(bin_dir) |
| 198 | + if exe is None or not _probe(exe)[0]: |
| 199 | + return {} |
| 200 | + # The subprocess runs outside the lock; a rare duplicate ctags run on |
| 201 | + # a concurrent query is cheaper than serializing every enrichment. |
| 202 | + tags = _run_ctags(exe, abs_path) |
| 203 | + with _lock: |
| 204 | + _tags_cache[key] = (signature, tags) |
| 205 | + _tags_cache.move_to_end(key) |
| 206 | + while len(_tags_cache) > CACHE_CAPACITY: |
| 207 | + _tags_cache.popitem(last=False) |
| 208 | + return tags |
| 209 | + |
| 210 | + |
| 211 | +def _kind_rank(kind: str | None) -> int: |
| 212 | + try: |
| 213 | + return _KIND_PRIORITY.index(kind) |
| 214 | + except ValueError: |
| 215 | + return len(_KIND_PRIORITY) |
| 216 | + |
| 217 | + |
| 218 | +def best_tag(tags: dict[str, list[dict]], name: str, line: int) -> dict | None: |
| 219 | + """The tag that matches a (name, line) result record, or None. |
| 220 | +
|
| 221 | + Exact line match first (kind-priority tie-break for multi-tag lines like |
| 222 | + ``typedef struct foo foo;``); when the name is unique in the file, accept |
| 223 | + it at any line — gtags' parser and ctags occasionally disagree by a line. |
| 224 | + """ |
| 225 | + entries = tags.get(name) |
| 226 | + if not entries: |
| 227 | + return None |
| 228 | + exact = [t for t in entries if t["line"] == line] |
| 229 | + if exact: |
| 230 | + return min(exact, key=lambda t: _kind_rank(t["kind"])) |
| 231 | + if len(entries) == 1: |
| 232 | + return entries[0] |
| 233 | + return None |
0 commit comments