Skip to content

Commit 7f8d1ac

Browse files
committed
Merge remote-tracking branch 'origin/main' into fix/cohort-lock-tmp-cleaner
2 parents 3cf2a66 + d615088 commit 7f8d1ac

14 files changed

Lines changed: 1788 additions & 21 deletions

‎internal/cbm/cbm.c‎

Lines changed: 242 additions & 9 deletions
Large diffs are not rendered by default.

‎internal/cbm/preprocessor.cpp‎

Lines changed: 304 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,283 @@ static bool has_preprocessor_work(const char *source, int source_len) {
4242
return false;
4343
}
4444

45+
// ── Export-macro candidates (#1989) ─────────────────────────────────────────
46+
// Build systems define symbol-export macros empty on the compiler command line
47+
// (UE's UBT: `/D "MODULE_API="`; CMake generate_export_header: `<lib>_EXPORT`),
48+
// so they never appear as #define lines in the source. tree-sitter has no
49+
// preprocessor state either, and `class MODULE_API Foo` misparses with the
50+
// macro token as the type name (worse: enums and free functions are lost to
51+
// ERROR regions entirely). To mirror the real compile line, the preprocessed
52+
// second pass predefines the conventional export-macro-shaped identifiers found
53+
// in the file as empty. The shape is deliberately narrow — ALL_CAPS identifier
54+
// ending in a known export suffix, with a non-trivial prefix — and the list is
55+
// capped, so ordinary all-caps identifiers cannot be swept in wholesale.
56+
static const char *kExportMacroSuffixes[] = {"_API", "_EXPORT", "_IMPORT",
57+
"_DLLEXPORT", "_DEPRECATED", NULL};
58+
59+
static bool is_export_macro_shape(const char *id, size_t len) {
60+
if (id[0] < 'A' || id[0] > 'Z') {
61+
return false;
62+
}
63+
for (size_t i = 0; i < len; i++) {
64+
char c = id[i];
65+
bool upper = c >= 'A' && c <= 'Z';
66+
bool digit = c >= '0' && c <= '9';
67+
if (!upper && !digit && c != '_') {
68+
return false;
69+
}
70+
}
71+
for (int s = 0; kExportMacroSuffixes[s]; s++) {
72+
size_t slen = strlen(kExportMacroSuffixes[s]);
73+
// Require at least two prefix characters before the suffix so a bare
74+
// "X_API"-style token (single-letter, easily a genuine symbol) stays out.
75+
if (len >= slen + 2 && strncmp(id + len - slen, kExportMacroSuffixes[s], slen) == 0) {
76+
return true;
77+
}
78+
}
79+
return false;
80+
}
81+
82+
static bool is_identifier_start(char c) {
83+
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_';
84+
}
85+
86+
static bool is_identifier_char(char c) {
87+
return is_identifier_start(c) || (c >= '0' && c <= '9');
88+
}
89+
90+
// C++ raw string literals (R"delim(...)delim"). Returns the index one past
91+
// the closing quote, or -1 when the form cannot be confidently recognized
92+
// (unterminated, malformed delimiter) — the caller then STOPS collecting so
93+
// the mis-modeled literal cannot poison the scan state of the rest of the
94+
// file (#1989 review).
95+
static int skip_raw_string(const char *source, int source_len, int quote) {
96+
int j = quote + 1;
97+
int delim_start = j;
98+
while (j < source_len) {
99+
char c = source[j];
100+
if (c == '(') {
101+
break;
102+
}
103+
// Delimiter chars: the standard d-char set excludes space, parens,
104+
// backslash, and the line breaks — a double quote IS a legal d-char
105+
// (R"""(...)""" compiles), so it must not end the delimiter (#1989
106+
// review round 3).
107+
if (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == ')' || c == '\\' ||
108+
j - delim_start >= 16) {
109+
return -1;
110+
}
111+
j++;
112+
}
113+
if (j >= source_len || source[j] != '(') {
114+
return -1;
115+
}
116+
int delim_len = j - delim_start;
117+
j++; // past the opening "("
118+
while (j < source_len) {
119+
if (source[j] == ')') {
120+
int k = j + 1;
121+
int m = 0;
122+
while (m < delim_len && k + m < source_len &&
123+
source[k + m] == source[delim_start + m]) {
124+
m++;
125+
}
126+
if (m == delim_len && k + m < source_len && source[k + m] == '"') {
127+
return k + m + 1;
128+
}
129+
}
130+
j++;
131+
}
132+
return -1; // unterminated raw string: uncertain
133+
}
134+
135+
// If the identifier just scanned is a raw-string prefix (R, LR, uR, UR, u8R)
136+
// immediately followed by a quote, return skip_raw_string(...). Returns 0 for
137+
// an ordinary identifier (including an identifier merely ENDING in R — only a
138+
// standalone prefix token starts a raw string) and -1 on an unrecognizable
139+
// raw form.
140+
static int raw_string_after_prefix(const char *source, int source_len, int start, int end) {
141+
static const char *prefixes[] = {"R", "LR", "uR", "UR", "u8R", NULL};
142+
size_t idlen = (size_t)(end - start);
143+
bool is_prefix = false;
144+
for (int p = 0; prefixes[p]; p++) {
145+
if (idlen == strlen(prefixes[p]) && strncmp(source + start, prefixes[p], idlen) == 0) {
146+
is_prefix = true;
147+
break;
148+
}
149+
}
150+
if (!is_prefix || end >= source_len || source[end] != '"') {
151+
return 0;
152+
}
153+
return skip_raw_string(source, source_len, end);
154+
}
155+
156+
// Advance past comments and string/char literals so only real code tokens
157+
// consume the candidate budget (#1989 review). Handles line-spliced //
158+
// comments (a backslash before the newline continues the comment — line
159+
// splicing happens before comment recognition) and, via the identifier-scan
160+
// hook in the callers, C++ raw string literals. A digit separator (1'000'000)
161+
// can still over-skip — that fails safe, toward NOT collecting a candidate.
162+
static int skip_non_code(const char *source, int source_len, int i) {
163+
while (i < source_len) {
164+
char c = source[i];
165+
if (c == '/' && i + 1 < source_len && source[i + 1] == '/') {
166+
i += 2;
167+
for (;;) {
168+
while (i < source_len && source[i] != '\n') {
169+
i++;
170+
}
171+
if (i >= source_len) {
172+
break;
173+
}
174+
// Line splice: a backslash right before the newline (allowing
175+
// \r\n) means the comment continues on the next line.
176+
int back = i - 1;
177+
if (back >= 0 && source[back] == '\r') {
178+
back--;
179+
}
180+
if (back >= 0 && source[back] == '\\') {
181+
i++; // spliced: keep consuming inside the comment
182+
continue;
183+
}
184+
i++; // real end-of-comment newline
185+
break;
186+
}
187+
} else if (c == '/' && i + 1 < source_len && source[i + 1] == '*') {
188+
i += 2;
189+
while (i + 1 < source_len && !(source[i] == '*' && source[i + 1] == '/')) {
190+
i++;
191+
}
192+
if (i + 1 < source_len) {
193+
i += 2; // past the closing "*/"
194+
} else {
195+
i = source_len; // unterminated comment runs to EOF
196+
}
197+
} else if (c == '"' || c == '\'') {
198+
char quote = c;
199+
i++;
200+
while (i < source_len) {
201+
if (source[i] == '\\') {
202+
i += 2;
203+
continue;
204+
}
205+
if (source[i] == quote) {
206+
i++;
207+
break;
208+
}
209+
i++;
210+
}
211+
} else {
212+
return i;
213+
}
214+
}
215+
return i;
216+
}
217+
218+
// Count-only scan used by the early gate (no storage).
219+
static bool has_export_macro_candidates(const char *source, int source_len) {
220+
if (!source || source_len <= 0) {
221+
return false;
222+
}
223+
int i = 0;
224+
while (i < source_len) {
225+
i = skip_non_code(source, source_len, i);
226+
if (i >= source_len) {
227+
break;
228+
}
229+
if (!is_identifier_start(source[i])) {
230+
i++;
231+
continue;
232+
}
233+
int start = i;
234+
while (i < source_len && is_identifier_char(source[i])) {
235+
i++;
236+
}
237+
// A raw-string prefix consumes everything up to its closing quote;
238+
// an unrecognizable raw form poisons the scan state, so fail toward
239+
// NOT running the second pass (raw behavior preserved).
240+
int raw = raw_string_after_prefix(source, source_len, start, i);
241+
if (raw < 0) {
242+
return false;
243+
}
244+
if (raw > 0) {
245+
i = raw;
246+
continue;
247+
}
248+
if (is_export_macro_shape(source + start, (size_t)(i - start))) {
249+
return true;
250+
}
251+
}
252+
return false;
253+
}
254+
255+
static int collect_export_macro_candidates(const char *source, int source_len,
256+
char (*out)[CBM_EXPORT_MACRO_NAME_MAX], int max_out) {
257+
if (!source || source_len <= 0 || !out || max_out <= 0) {
258+
return 0;
259+
}
260+
int stored = 0;
261+
int i = 0;
262+
while (i < source_len) {
263+
i = skip_non_code(source, source_len, i);
264+
if (i >= source_len) {
265+
break;
266+
}
267+
if (!is_identifier_start(source[i])) {
268+
i++;
269+
continue;
270+
}
271+
int start = i;
272+
while (i < source_len && is_identifier_char(source[i])) {
273+
i++;
274+
}
275+
// Raw string (see has_export_macro_candidates): skip its body, or stop
276+
// collecting entirely when the form cannot be confidently recognized —
277+
// continuing would mis-tokenize the rest of the file and could both
278+
// mint phantom candidates and mask real ones (#1989 review).
279+
int raw = raw_string_after_prefix(source, source_len, start, i);
280+
if (raw < 0) {
281+
break;
282+
}
283+
if (raw > 0) {
284+
i = raw;
285+
continue;
286+
}
287+
size_t len = (size_t)(i - start);
288+
/* Length gate FIRST: the dedup below reads out[k][0..len] against rows
289+
* of CBM_EXPORT_MACRO_NAME_MAX bytes — an over-long candidate must be
290+
* rejected before any compare touches them. */
291+
if (len >= CBM_EXPORT_MACRO_NAME_MAX) {
292+
continue;
293+
}
294+
if (!is_export_macro_shape(source + start, len)) {
295+
continue;
296+
}
297+
bool dup = false;
298+
for (int k = 0; k < stored; k++) {
299+
if (strncmp(out[k], source + start, len) == 0 && out[k][len] == '\0') {
300+
dup = true;
301+
break;
302+
}
303+
}
304+
if (dup) {
305+
continue;
306+
}
307+
if (stored >= max_out) {
308+
break; // bounded: stop collecting once the cap is reached
309+
}
310+
memcpy(out[stored], source + start, len);
311+
out[stored][len] = '\0';
312+
stored++;
313+
}
314+
return stored;
315+
}
316+
317+
int cbm_export_macro_candidates(const char *source, int source_len,
318+
char (*out)[CBM_EXPORT_MACRO_NAME_MAX], int max_out) {
319+
return collect_export_macro_candidates(source, source_len, out, max_out);
320+
}
321+
45322
static int count_expanded_lines(const std::string &text) {
46323
int count = 1;
47324
for (char c : text) {
@@ -146,7 +423,11 @@ static bool build_line_map(const std::string &expanded, const std::string &main_
146423
CBMPreprocessedSource *cbm_preprocess_with_map(const char *source, int source_len,
147424
const char *filename, const char **extra_defines,
148425
const char **include_paths, int cpp_mode) {
149-
if (!has_preprocessor_work(source, source_len)) {
426+
// Run the second pass when there are directives to evaluate OR when the file
427+
// carries export-macro-shaped identifiers to predefine empty (#1989) — a UE
428+
// plugin header with only `#pragma once` + `#include` lines still needs it.
429+
if (!has_preprocessor_work(source, source_len) &&
430+
!has_export_macro_candidates(source, source_len)) {
150431
return NULL; // NULL = no expansion needed, use original
151432
}
152433

@@ -156,6 +437,28 @@ CBMPreprocessedSource *cbm_preprocess_with_map(const char *source, int source_le
156437
for (int i = 0; extra_defines[i]; i++)
157438
dui.defines.push_back(extra_defines[i]);
158439
}
440+
// Predefine collected export-macro candidates as empty, mirroring the
441+
// real compile command line (`/D "MODULE_API="`). Names already provided
442+
// by the caller win — never override an explicit define.
443+
char export_cands[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX];
444+
int export_cand_count =
445+
collect_export_macro_candidates(source, source_len, export_cands, CBM_EXPORT_MACRO_MAX);
446+
for (int i = 0; i < export_cand_count; i++) {
447+
bool provided = false;
448+
for (std::list<std::string>::const_iterator it = dui.defines.begin();
449+
it != dui.defines.end(); ++it) {
450+
const std::string &def = *it;
451+
size_t eq = def.find('=');
452+
std::string name = (eq == std::string::npos) ? def : def.substr(0, eq);
453+
if (name == export_cands[i]) {
454+
provided = true;
455+
break;
456+
}
457+
}
458+
if (!provided) {
459+
dui.defines.push_back(std::string(export_cands[i]) + "=");
460+
}
461+
}
159462
if (include_paths) {
160463
for (int i = 0; include_paths[i]; i++)
161464
dui.includePaths.push_back(include_paths[i]);

‎internal/cbm/preprocessor.h‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,15 @@
44
#include <stddef.h>
55
#include <stdint.h>
66

7+
/* #1989: build-system export macros (UE "<MOD>_API", CMake generate_export_header
8+
* "<lib>_EXPORT", ...) are defined empty on the real compile command line but are
9+
* invisible to tree-sitter, so `class MOD_API Foo` misparses with the macro token
10+
* as the type name. The preprocessed second pass predefines the conventional
11+
* export-macro-shaped identifiers found in a source file as empty, mirroring the
12+
* real build line. The candidate list is bounded. */
13+
#define CBM_EXPORT_MACRO_MAX 32
14+
#define CBM_EXPORT_MACRO_NAME_MAX 96
15+
716
#ifdef __cplusplus
817
extern "C" {
918
#endif
@@ -15,6 +24,12 @@ typedef struct {
1524
int expanded_line_count;
1625
} CBMPreprocessedSource;
1726

27+
// Collect conventional export-macro-shaped identifiers (ALL_CAPS ending in
28+
// _API/_EXPORT/_IMPORT/_DLLEXPORT/_DEPRECATED) from the source, up to max_out
29+
// unique names. Returns the number of names stored in out.
30+
int cbm_export_macro_candidates(const char *source, int source_len,
31+
char (*out)[CBM_EXPORT_MACRO_NAME_MAX], int max_out);
32+
1833
// Preprocess C/C++ source: expand macros, evaluate #ifdef, resolve #include.
1934
// Returns malloc-allocated expanded source, or NULL if no expansion needed/on failure.
2035
// extra_defines: NULL-terminated array of "NAME=VALUE" strings (can be NULL).

0 commit comments

Comments
 (0)