Skip to content

Commit 79b29fc

Browse files
authored
Merge pull request DeusData#2543 from DeusData/fix/c-typedefs-macros-enumerators
fix(extract): C typedefs become nodes; macros and enumerators get their own QNs
2 parents d1088fd + 8a97baa commit 79b29fc

24 files changed

Lines changed: 3100 additions & 124 deletions

‎internal/cbm/cbm.c‎

Lines changed: 465 additions & 0 deletions
Large diffs are not rendered by default.

‎internal/cbm/cbm.h‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,18 @@ typedef struct {
267267
* HTTP_CALLS edge to base + path. Tail fields: zero-init stays valid. */
268268
const char *http_client;
269269
const char *http_base_url;
270+
/* Set when this FILE holds two or more definitions with this QN and label
271+
* (`#if`/`#else` twins, a macro redefined per platform, overloads): the
272+
* graph keeps one node per QN, and this lists every one of those
273+
* definitions' line spans, the surviving one included, as a JSON array
274+
* sorted by start line, in graph_buffer.c's variants schema:
275+
* [{"file_path":"a.c","start_line":10,"end_line":14},
276+
* {"file_path":"a.c","start_line":20,"end_line":26}].
277+
* Carried by the definition the graph keeps for the file (the last by
278+
* start line), which makes it the node's `variants` property. NULL on
279+
* every other definition, and in every language outside
280+
* cbm_is_c_preprocessor_lang. */
281+
const char *variants;
270282
/* Callable identity (#2061): offset of the signature suffix inside
271283
* qualified_name (base QN = the first qn_sig_off bytes); 0 = no suffix.
272284
* Always 0 until a language enables its callable_identity mode. */
@@ -288,6 +300,17 @@ typedef struct {
288300

289301
#define CBM_MAX_CALL_ARGS 8
290302

303+
/* The qualified name of a C-preprocessor macro (`#define NAME ...` in C, C++,
304+
* CUDA, Objective-C, GLSL, ISPC) is `<module QN>.<NAME>#macro`. C keeps macros
305+
* apart from ordinary identifiers, so a macro and a function, type, variable or
306+
* enumerator may legally share a name in one file; the fence gives the macro
307+
* an identity of its own, and the plain `<module QN>.<NAME>` always belongs to
308+
* the definition. `name` stays NAME and the label stays "Macro".
309+
*
310+
* Tie rule for anything that resolves a NAME or a plain QN to a node: the
311+
* definition is the target, and the macro only when no definition is visible. */
312+
#define CBM_MACRO_QN_SUFFIX "#macro"
313+
291314
/* Byte offsets are meaningful only within the source buffer that produced
292315
* them. C/C++/CUDA run both raw and preprocessed extraction passes, and those
293316
* buffers can contain unrelated occurrences at the same numeric span. */

‎internal/cbm/extract_defs.c‎

Lines changed: 588 additions & 55 deletions
Large diffs are not rendered by default.

‎internal/cbm/helpers.c‎

Lines changed: 130 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -988,16 +988,109 @@ static TSNode resolve_qualified_name(TSNode decl) {
988988
return null_node;
989989
}
990990

991+
bool cbm_c_qualifier_is_recovered(TSNode qid) {
992+
if (ts_node_is_null(qid)) {
993+
return false;
994+
}
995+
uint32_t nc = ts_node_child_count(qid);
996+
for (uint32_t i = 0; i < nc; i++) {
997+
TSNode c = ts_node_child(qid, i);
998+
if (!ts_node_is_named(c) && strcmp(ts_node_type(c), "::") == 0) {
999+
return ts_node_is_missing(c);
1000+
}
1001+
}
1002+
return false;
1003+
}
1004+
1005+
/* A function DEFINITION's parameter list: empty, `(void)`, or every parameter
1006+
* named (variadic `...` allowed). A function-like macro invocation that only
1007+
* looks like a declarator — `TEST_BEGIN(test_name)`, whose one "parameter" is
1008+
* an untyped identifier read as a type — fails this, so its macro name is never
1009+
* mistaken for the defined function's name. */
1010+
static bool c_params_name_every_parameter(TSNode params) {
1011+
enum { MAX_PARAMS_CHECKED = 64 }; /* indexed child access: keep it small */
1012+
uint32_t nc = ts_node_named_child_count(params);
1013+
if (nc > MAX_PARAMS_CHECKED) {
1014+
return false;
1015+
}
1016+
for (uint32_t i = 0; i < nc; i++) {
1017+
TSNode p = ts_node_named_child(params, i);
1018+
const char *pk = ts_node_type(p);
1019+
if (strcmp(pk, "comment") == 0 || strcmp(pk, "variadic_parameter") == 0) {
1020+
continue;
1021+
}
1022+
if (!ts_node_is_null(ts_node_child_by_field_name(p, TS_FIELD("declarator")))) {
1023+
continue;
1024+
}
1025+
/* `(void)`: the one parameter is an unnamed primitive type. */
1026+
TSNode type = ts_node_child_by_field_name(p, TS_FIELD("type"));
1027+
if (nc != 1 || ts_node_is_null(type) || strcmp(ts_node_type(type), "primitive_type") != 0) {
1028+
return false;
1029+
}
1030+
}
1031+
return true;
1032+
}
1033+
1034+
TSNode cbm_c_recovered_func_name(TSNode func_declarator) {
1035+
TSNode null_node = {0};
1036+
if (ts_node_is_null(func_declarator) ||
1037+
strcmp(ts_node_type(func_declarator), "function_declarator") != 0) {
1038+
return null_node;
1039+
}
1040+
TSNode params = ts_node_child_by_field_name(func_declarator, TS_FIELD("parameters"));
1041+
if (ts_node_is_null(params) || !c_params_name_every_parameter(params)) {
1042+
return null_node;
1043+
}
1044+
TSNode err = ts_node_prev_sibling(params);
1045+
if (ts_node_is_null(err) || strcmp(ts_node_type(err), "ERROR") != 0) {
1046+
return null_node;
1047+
}
1048+
/* Only the macro-prefix shape: the ERROR holds nothing but bare identifiers
1049+
* (the real name, possibly after further attribute macros). Any other token
1050+
* means the region is not a declaration this rule can vouch for. */
1051+
enum { MAX_PREFIX_TOKENS = 4 };
1052+
uint32_t nc = ts_node_child_count(err);
1053+
if (nc == 0 || nc > MAX_PREFIX_TOKENS) {
1054+
return null_node;
1055+
}
1056+
TSNode last = null_node;
1057+
for (uint32_t i = 0; i < nc; i++) {
1058+
TSNode c = ts_node_child(err, i);
1059+
const char *ck = ts_node_type(c);
1060+
if (strcmp(ck, "identifier") != 0 && strcmp(ck, "type_identifier") != 0) {
1061+
return null_node;
1062+
}
1063+
last = c;
1064+
}
1065+
return last;
1066+
}
1067+
9911068
// Resolve function name from C/C++/CUDA/GLSL declarator chain. Shared canonical
9921069
// implementation — see the header for the full rationale (#438).
9931070
TSNode cbm_resolve_c_declarator_name_node(TSNode func_node) {
9941071
TSNode decl = ts_node_child_by_field_name(func_node, TS_FIELD("declarator"));
1072+
/* Set once a recovered `RetT::name` qualifier was stepped through: on that
1073+
* path the C++ grammar names the function with a type_identifier. */
1074+
bool recovered = false;
9951075
for (int depth = 0; depth < CBM_DECLARATOR_DEPTH_LIMIT && !ts_node_is_null(decl); depth++) {
9961076
const char *dk = ts_node_type(decl);
997-
if (is_c_terminal_name(dk)) {
1077+
if (is_c_terminal_name(dk) || (recovered && strcmp(dk, "type_identifier") == 0)) {
9981078
return decl;
9991079
}
1080+
if (strcmp(dk, "function_declarator") == 0) {
1081+
TSNode real = cbm_c_recovered_func_name(decl);
1082+
if (!ts_node_is_null(real)) {
1083+
return real;
1084+
}
1085+
}
10001086
if (strcmp(dk, "qualified_identifier") == 0 || strcmp(dk, "scoped_identifier") == 0) {
1087+
if (cbm_c_qualifier_is_recovered(decl)) {
1088+
/* `API RetT name(...)`: the "scope" is the return type, the name
1089+
* side carries the real declarator chain. */
1090+
decl = ts_node_child_by_field_name(decl, TS_FIELD("name"));
1091+
recovered = true;
1092+
continue;
1093+
}
10011094
return resolve_qualified_name(decl);
10021095
}
10031096
TSNode inner = ts_node_child_by_field_name(decl, TS_FIELD("declarator"));
@@ -1013,6 +1106,36 @@ TSNode cbm_resolve_c_declarator_name_node(TSNode func_node) {
10131106
return null_node;
10141107
}
10151108

1109+
/* The C-declarator grammars: same set resolve_func_name_c_family routes through
1110+
* cbm_resolve_c_declarator_name_node. */
1111+
static bool c_declarator_lang(CBMLanguage lang) {
1112+
return lang == CBM_LANG_C || lang == CBM_LANG_CPP || lang == CBM_LANG_CUDA ||
1113+
lang == CBM_LANG_GLSL || lang == CBM_LANG_HLSL || lang == CBM_LANG_ISPC ||
1114+
lang == CBM_LANG_SLANG || lang == CBM_LANG_OBJC;
1115+
}
1116+
1117+
bool cbm_c_reserved_func_name(const char *name) {
1118+
/* Statement/operator keywords: error recovery over preprocessor-split code
1119+
* reads `else if (a) (b) {` as a definition `else if(...) {...}`. */
1120+
static const char *const kw[] = {"if", "else", "for", "while", "do",
1121+
"switch", "case", "default", "return", "break",
1122+
"continue", "goto", "sizeof", "typedef", NULL};
1123+
if (!name) {
1124+
return false;
1125+
}
1126+
for (const char *const *k = kw; *k; k++) {
1127+
if (strcmp(name, *k) == 0) {
1128+
return true;
1129+
}
1130+
}
1131+
return false;
1132+
}
1133+
1134+
bool cbm_is_c_preprocessor_lang(CBMLanguage lang) {
1135+
return lang == CBM_LANG_C || lang == CBM_LANG_CPP || lang == CBM_LANG_CUDA ||
1136+
lang == CBM_LANG_GLSL || lang == CBM_LANG_OBJC || lang == CBM_LANG_ISPC;
1137+
}
1138+
10161139
// Convert a resolved function/method name node to its name string. Most nodes
10171140
// map directly to their text, but a C++ conversion-operator's `operator_cast`
10181141
// node spans the full "operator bool() const" — this grammar folds the parameter
@@ -1023,6 +1146,12 @@ TSNode cbm_resolve_c_declarator_name_node(TSNode func_node) {
10231146
// `if (obj)`) misses.
10241147
char *cbm_func_name_node_text(CBMArena *a, TSNode name_node, const char *source, CBMLanguage lang) {
10251148
char *text = cbm_node_text(a, name_node, source);
1149+
/* A C keyword is never a function name: the definition is an error-recovery
1150+
* artifact. No name means no def and no call scope (calls inside fall back
1151+
* to the enclosing scope), the same as any unnamed definition. */
1152+
if (text && c_declarator_lang(lang) && cbm_c_reserved_func_name(text)) {
1153+
return NULL;
1154+
}
10261155
if (text && strcmp(ts_node_type(name_node), "operator_cast") == 0) {
10271156
char *paren = strchr(text, '(');
10281157
if (paren) {

‎internal/cbm/helpers.h‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,37 @@ const char *cbm_enclosing_func_qn_cached(CBMExtractCtx *ctx, TSNode node);
5858
// enclosing-function attribution — drift between private copies caused #438.
5959
TSNode cbm_resolve_c_declarator_name_node(TSNode func_node);
6060

61+
// tree-sitter error recovery around an unknown leading macro: `API RetT name(...)`
62+
// (`XXH_PUBLIC_API XXH64_hash_t XXH64(...)`, `JEMALLOC_ALWAYS_INLINE T *f(...)`)
63+
// carries one identifier more than the declaration grammar admits, so the parser
64+
// recovers in one of two shapes, and neither names the function:
65+
// - C++ inserts a zero-width MISSING "::" and reads `RetT name` as the
66+
// out-of-line name `RetT::name`; with a pointer return, the name side is a
67+
// pointer_type_declarator that wraps the function_declarator;
68+
// - C and C++ keep RetT as the declarator identifier and put the real name in
69+
// an ERROR node directly before the parameter list.
70+
// cbm_c_qualifier_is_recovered: true when `qid` (a qualified_identifier or
71+
// scoped_identifier) is the first shape, i.e. its "::" token is MISSING.
72+
// cbm_c_recovered_func_name: the real name node of the second shape for a
73+
// function_declarator, or a null node when the declarator is not that shape.
74+
// Shared by every C-family declarator walk (defs, call scope, params, C LSP) so
75+
// the def QN and every caller QN agree on the recovered name.
76+
bool cbm_c_qualifier_is_recovered(TSNode qid);
77+
TSNode cbm_c_recovered_func_name(TSNode func_declarator);
78+
79+
// True for a C keyword that error recovery can surface as a function "name"
80+
// (`else if (a) (b) {` split by #ifdef reads as a definition named `if`).
81+
// cbm_func_name_node_text returns NULL for these in the C-declarator grammars,
82+
// so neither a def nor a call scope is minted; the C LSP applies the same test.
83+
bool cbm_c_reserved_func_name(const char *name);
84+
85+
// True for the languages whose `#define`s become Macro nodes: C, C++, CUDA,
86+
// GLSL, Objective-C and ISPC. The one predicate for what follows from the C
87+
// preprocessor: the `#macro` QN namespace (CBM_MACRO_QN_SUFFIX) and the
88+
// `variants` property (CBMDefinition.variants), because #if branches are what
89+
// lets one of these files define a name more than once.
90+
bool cbm_is_c_preprocessor_lang(CBMLanguage lang);
91+
6192
// Convert a resolved function/method name node to its name string, normalizing a
6293
// C++ conversion-operator's `operator_cast` node (which spans the full
6394
// "operator bool() const") down to "operator bool". Shared by the defs and

‎internal/cbm/lsp/c_lsp.c‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5349,22 +5349,44 @@ static void c_process_function(CLSPContext *ctx, TSNode func_node) {
53495349

53505350
// Navigate declarator to find name and parameters
53515351
TSNode cur = decl;
5352+
// Set after stepping through a recovered `RetT::name` (see
5353+
// cbm_c_qualifier_is_recovered): that path names the function with a
5354+
// type_identifier, and the "scope" is the return type, not a class.
5355+
bool recovered = false;
53525356
for (int depth = 0; depth < 10 && !ts_node_is_null(cur); depth++) {
53535357
const char *dk = ts_node_type(cur);
53545358

53555359
if (strcmp(dk, "function_declarator") == 0) {
53565360
TSNode fdecl = ts_node_child_by_field_name(cur, "declarator", 10);
53575361
params_node = ts_node_child_by_field_name(cur, "parameters", 10);
5362+
// `API RetT name(...)`: the real name sits in an ERROR before the
5363+
// parameters; the declarator identifier is the return type.
5364+
TSNode real = cbm_c_recovered_func_name(cur);
5365+
if (!ts_node_is_null(real)) {
5366+
func_name = c_node_text(ctx, real);
5367+
break;
5368+
}
53585369
cur = fdecl;
53595370
continue;
53605371
}
5361-
if (strcmp(dk, "pointer_declarator") == 0 || strcmp(dk, "reference_declarator") == 0) {
5372+
if (strcmp(dk, "pointer_declarator") == 0 || strcmp(dk, "reference_declarator") == 0 ||
5373+
(recovered && strcmp(dk, "pointer_type_declarator") == 0)) {
53625374
if (ts_node_named_child_count(cur) > 0)
53635375
cur = ts_node_named_child(cur, ts_node_named_child_count(cur) - 1);
53645376
else
53655377
break;
53665378
continue;
53675379
}
5380+
if (recovered && strcmp(dk, "type_identifier") == 0) {
5381+
func_name = c_node_text(ctx, cur);
5382+
break;
5383+
}
5384+
if ((strcmp(dk, "qualified_identifier") == 0 || strcmp(dk, "scoped_identifier") == 0) &&
5385+
cbm_c_qualifier_is_recovered(cur)) {
5386+
cur = ts_node_child_by_field_name(cur, "name", 4);
5387+
recovered = true;
5388+
continue;
5389+
}
53685390
if (strcmp(dk, "qualified_identifier") == 0 || strcmp(dk, "scoped_identifier") == 0) {
53695391
func_name = c_node_text(ctx, cur);
53705392
// Check if this is a method (has Class:: prefix)
@@ -5404,7 +5426,9 @@ static void c_process_function(CLSPContext *ctx, TSNode func_node) {
54045426
break;
54055427
}
54065428

5407-
if (!func_name || !func_name[0])
5429+
// A keyword "name" is an error-recovery artifact (see cbm_c_reserved_func_name):
5430+
// the def extractor mints no node for it, so no caller QN may name it either.
5431+
if (!func_name || !func_name[0] || cbm_c_reserved_func_name(func_name))
54085432
return;
54095433

54105434
const char *func_qn = cbm_test_definition_owner_qn(ctx->test_definition_owners, func_node);

‎internal/cbm/result_compact.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,7 @@ static void cr_walk_def(cr_ctx_t *c, CBMDefinition *d) {
275275
cr_str(c, &d->impl_trait);
276276
cr_str(c, &d->http_client);
277277
cr_str(c, &d->http_base_url);
278+
cr_str(c, &d->variants);
278279
}
279280

280281
static void cr_walk_call(cr_ctx_t *c, CBMCall *call) {

‎src/graph_buffer/graph_buffer.c‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -993,6 +993,20 @@ static void gb_record_variants(cbm_gbuf_node_t *survivor, const char *a_label, c
993993

994994
/* ── Node operations ─────────────────────────────────────────────── */
995995

996+
/* First key of the canonical same-QN collision order: how strong a claim the
997+
* entity has on a qualified name. A macro is the weakest: when a "Macro" and
998+
* a definition share a QN, the definition, which carries the body, the
999+
* signature and every edge, owns it whichever comes first.
1000+
*
1001+
* C-preprocessor macros no longer get here: their QN carries its own
1002+
* namespace (`...#macro`, CBM_MACRO_QN_SUFFIX) and cannot equal a definition's.
1003+
* The case left is Chialisp, whose `defmacro` / `defmac` defs are labelled
1004+
* "Macro" under a plain QN: a macro and a `defun` of one name in one file, or
1005+
* in a same-stem `.clsp` / `.clib` pair (one module QN). */
1006+
static int gb_qn_claim_rank(const char *label) {
1007+
return (label && strcmp(label, "Macro") == 0) ? 0 : 1;
1008+
}
1009+
9961010
int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name,
9971011
const char *qualified_name, const char *file_path, int start_line,
9981012
int end_line, const char *properties_json) {
@@ -1036,8 +1050,13 @@ int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name
10361050
* of the same entity with new content) replaces the earlier one,
10371051
* deterministically, because intra-file arrival order is fixed. A
10381052
* full tie is the same entity re-upserted → refresh in place.
1039-
* Kind-disambiguated QNs (the real cure) remain a follow-up. */
1040-
int c = strcmp(file_path ? file_path : "", existing->file_path ? existing->file_path : "");
1053+
* Ahead of all of these, a definition beats a macro
1054+
* (gb_qn_claim_rank). Kind-disambiguated QNs (the real cure) remain
1055+
* a follow-up. */
1056+
int c = gb_qn_claim_rank(existing->label) - gb_qn_claim_rank(label);
1057+
if (c == 0) {
1058+
c = strcmp(file_path ? file_path : "", existing->file_path ? existing->file_path : "");
1059+
}
10411060
if (c == 0) {
10421061
c = existing->start_line - start_line;
10431062
}
@@ -1649,9 +1668,13 @@ static void merge_update_existing(cbm_gbuf_t *dst, cbm_gbuf_node_t *existing,
16491668
* the node set (and every downstream consumer) run to run. Winner =
16501669
* smallest file_path, then LARGEST start_line, then largest
16511670
* name/label — one total order, commutative, scheduling-free; a full
1652-
* tie is the same entity → refresh from src. */
1653-
int c = strcmp(sn->file_path ? sn->file_path : "",
1671+
* tie is the same entity → refresh from src. A definition beats a
1672+
* macro ahead of all of these (gb_qn_claim_rank). */
1673+
int c = gb_qn_claim_rank(existing->label) - gb_qn_claim_rank(sn->label);
1674+
if (c == 0) {
1675+
c = strcmp(sn->file_path ? sn->file_path : "",
16541676
existing->file_path ? existing->file_path : "");
1677+
}
16551678
if (c == 0) {
16561679
c = existing->start_line - sn->start_line;
16571680
}

‎src/mcp/mcp.c‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8557,6 +8557,17 @@ static long node_resolution_score(const cbm_node_t *n) {
85578557
label_rank = RES_RANK_OTHER;
85588558
}
85598559
}
8560+
/* Tie rule (CBM_MACRO_QN_SUFFIX): a name that is both a definition and a C
8561+
* macro -- a typedef or enumerator next to its rename macro -- resolves to
8562+
* the definition. The macro scores just under every other definition, and
8563+
* still above Module/File; without this a one-line typedef and its macro
8564+
* tie on span and the name reads as ambiguous. */
8565+
size_t qn_len = n->qualified_name ? strlen(n->qualified_name) : 0;
8566+
size_t fence_len = sizeof(CBM_MACRO_QN_SUFFIX) - SKIP_ONE;
8567+
if (label_rank == RES_RANK_OTHER && qn_len > fence_len &&
8568+
strcmp(n->qualified_name + qn_len - fence_len, CBM_MACRO_QN_SUFFIX) == 0) {
8569+
return RES_RANK_OTHER * (long)RES_LABEL_WEIGHT - SKIP_ONE;
8570+
}
85608571
long span = (long)n->end_line - (long)n->start_line;
85618572
if (span < 0) {
85628573
span = 0;
@@ -13101,6 +13112,23 @@ static char *handle_get_code_snippet(cbm_mcp_server_t *srv, const char *args) {
1310113112
result =
1310213113
snippet_from_tier(srv, tier_nodes, tier_count, qn, "suffix", include_neighbors, args);
1310313114
}
13115+
13116+
/* Tier 3: the C-macro namespace. A macro's QN is `<module>.<NAME>#macro`
13117+
* (CBM_MACRO_QN_SUFFIX), out of reach of '%.X'. Tried only when no
13118+
* definition answered to the name above (tie rule), for a short name, a
13119+
* partial QN, or the macro's plain QN. */
13120+
if (!result) {
13121+
char macro_qn[CBM_SZ_512];
13122+
int macro_len = snprintf(macro_qn, sizeof(macro_qn), "%s" CBM_MACRO_QN_SUFFIX, qn);
13123+
if (macro_len > 0 && (size_t)macro_len < sizeof(macro_qn)) {
13124+
tier_nodes = NULL;
13125+
tier_count = 0;
13126+
cbm_store_find_nodes_by_qn_suffix(store, effective_project, macro_qn, &tier_nodes,
13127+
&tier_count);
13128+
result = snippet_from_tier(srv, tier_nodes, tier_count, qn, "suffix", include_neighbors,
13129+
args);
13130+
}
13131+
}
1310413132
free(qn);
1310513133
free(project);
1310613134
if (result) {

0 commit comments

Comments
 (0)