Skip to content

Commit 09dc36d

Browse files
fix(extract): parse PHP past inline HTML so symbols after ?> reach the graph (DeusData#2000)
Signed-off-by: harshitaajoshi <j.harshitaa06@gmail.com>
1 parent 4c80d89 commit 09dc36d

7 files changed

Lines changed: 528 additions & 0 deletions

File tree

‎Makefile.cbm‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -398,6 +398,7 @@ EXTRACTION_SRCS = \
398398
$(CBM_DIR)/route_path.c \
399399
$(CBM_DIR)/openapi.c \
400400
$(CBM_DIR)/sql_values.c \
401+
$(CBM_DIR)/php_inline_html.c \
401402
$(CBM_DIR)/helpers.c \
402403
$(CBM_DIR)/cpp_branch_views.c \
403404
$(CBM_DIR)/result_compact.c \

‎internal/cbm/cbm.c‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3313,6 +3313,18 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage
33133313
return r;
33143314
}
33153315

3316+
/* The bytes the grammar parses. PHP that leaves PHP mode (`?> markup <?php`)
3317+
* is parsed with its markup blanked (#2000); the rewrite keeps the length and
3318+
* every line break, so offsets and line numbers still refer to the file on
3319+
* disk. Every other language parses the file as it is. */
3320+
static const char *extract_parse_bytes(CBMArena *a, CBMLanguage language, const char *source,
3321+
int source_len) {
3322+
if (language == CBM_LANG_PHP) {
3323+
return cbm_php_mask_inline_html(a, source, source_len);
3324+
}
3325+
return source;
3326+
}
3327+
33163328
/* Initial block for the per-file traversal scratch arena, chosen by measuring
33173329
* arena_grow on a 14k-file TypeScript tree: it fires on one file in 12,000 at
33183330
* both this size and at 1 MB, and on most files at 256 KB, where the two
@@ -3733,6 +3745,8 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
37333745
return result;
37343746
}
37353747

3748+
source = extract_parse_bytes(a, language, source, source_len);
3749+
37363750
// Skip pathologically nested Perl before tree-sitter's recursive GLR stack
37373751
// merge overflows a small stack during the parse (see
37383752
// cbm_source_nesting_exceeds). Scoped to Perl: its ambiguous call grammar is

‎internal/cbm/cbm.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1361,6 +1361,10 @@ void cbm_extract_definitions_without_module(CBMExtractCtx *ctx);
13611361
// per ref()/source() call. No-op unless the file parses as SQL and actually
13621362
// contains a dbt builtin call. Defined in extract_dbt.c.
13631363
void cbm_extract_dbt(CBMExtractCtx *ctx);
1364+
// PHP source with its inline-HTML regions blanked so the php_only grammar can
1365+
// parse past `?>` (#2000). Same length and line breaks as the input; returns
1366+
// `source` itself when nothing needs rewriting. Defined in php_inline_html.c.
1367+
const char *cbm_php_mask_inline_html(CBMArena *arena, const char *source, int source_len);
13641368
void cbm_extract_imports(CBMExtractCtx *ctx);
13651369
void cbm_extract_usages(CBMExtractCtx *ctx);
13661370
void cbm_extract_semantic(CBMExtractCtx *ctx);

‎internal/cbm/lsp/php_lsp.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "php_lsp.h"
2121
#include "lsp_node_iter.h"
2222
#include "lsp_work.h"
23+
#include "../cbm.h"
2324
#include "../helpers.h"
2425
#include "../../../src/foundation/hash_table.h"
2526
#include <ctype.h>
@@ -4795,6 +4796,9 @@ void cbm_run_php_lsp_cross_with_bindings(CBMArena *arena, const char *source, in
47954796
const CBMPHPImportBindings *bindings) {
47964797
if (!arena || !source || source_len <= 0 || !out)
47974798
return;
4799+
/* The retained source is the file on disk; read it the way the per-file
4800+
* extract did, so the cached tree and node text agree (#2000). */
4801+
source = cbm_php_mask_inline_html(arena, source, source_len);
47984802

47994803
TSParser *parser = NULL;
48004804
TSTree *tree = cached_tree;

‎internal/cbm/php_inline_html.c‎

Lines changed: 317 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,317 @@
1+
// php_inline_html.c — let the php_only grammar read a file that leaves PHP mode.
2+
//
3+
// A PHP file starts in inline-HTML mode and may switch in and out of PHP any
4+
// number of times: `?> <div>...</div> <?php`. The vendored grammar is the
5+
// tree-sitter-php `php_only` variant, which has no rule for `?>` or for the
6+
// text that follows it, so every byte after the first close tag used to land in
7+
// an ERROR region and the declarations there never reached the graph (#2000).
8+
//
9+
// Instead of parsing the markup, blank it. The rewritten buffer has the same
10+
// length as the original and keeps every line break, so byte offsets, rows and
11+
// columns are unchanged and node text inside PHP code is byte-identical:
12+
//
13+
// - markup before the first open tag and between tags becomes spaces
14+
// - the first open tag stays, so the tree keeps its php_tag
15+
// - each later `<?php`, `<?=` or `<?` becomes spaces; after `<?=` the echoed
16+
// expression is left as an expression statement, so its calls are kept
17+
// - each `?>` becomes `; `, the statement terminator PHP itself treats it as
18+
//
19+
// Only code and `//` / `#` comments can close PHP mode. Strings, heredoc,
20+
// nowdoc and block comments are skipped, `#[` is an attribute, and everything
21+
// after __halt_compiler is left alone, as PHP stops reading there.
22+
23+
#include "cbm.h"
24+
#include "foundation/arena.h"
25+
#include "foundation/constants.h"
26+
#include <stdbool.h>
27+
#include <stddef.h>
28+
#include <string.h>
29+
30+
enum {
31+
PHP_TAG_OPEN_LEN = 5, /* <?php */
32+
PHP_TAG_ECHO_LEN = 3, /* <?= */
33+
PHP_TAG_SHORT_LEN = 2, /* <? */
34+
PHP_HEREDOC_LEN = 3, /* <<< */
35+
PHP_FIRST_NON_ASCII = 0x80,
36+
PHP_NOT_FOUND = -1,
37+
};
38+
39+
static const char PHP_HALT_COMPILER[] = "__halt_compiler";
40+
enum { PHP_HALT_COMPILER_LEN = (int)sizeof(PHP_HALT_COMPILER) - SKIP_ONE };
41+
42+
/* True when the literal `lit` starts at s[i]. */
43+
static bool php_at(const char *s, int len, int i, const char *lit) {
44+
size_t n = strlen(lit);
45+
return i >= 0 && i <= len && (size_t)(len - i) >= n && memcmp(s + i, lit, n) == 0;
46+
}
47+
48+
static bool php_is_ident_char(unsigned char c) {
49+
return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ||
50+
c >= PHP_FIRST_NON_ASCII;
51+
}
52+
53+
static bool php_is_ident_start(unsigned char c) {
54+
return php_is_ident_char(c) && !(c >= '0' && c <= '9');
55+
}
56+
57+
static bool php_is_space(unsigned char c) {
58+
return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v';
59+
}
60+
61+
static bool php_is_blank(char c) {
62+
return c == ' ' || c == '\t';
63+
}
64+
65+
/* Case-insensitive match of the lowercase `word` at s[i]. */
66+
static bool php_ieq(const char *s, int len, int i, const char *word) {
67+
int n = (int)strlen(word);
68+
if (i + n > len) {
69+
return false;
70+
}
71+
for (int k = 0; k < n; k++) {
72+
char c = s[i + k];
73+
if (c >= 'A' && c <= 'Z') {
74+
c = (char)(c - 'A' + 'a');
75+
}
76+
if (c != word[k]) {
77+
return false;
78+
}
79+
}
80+
return true;
81+
}
82+
83+
/* Length of the open tag at i, or 0. `<?php` needs whitespace or EOF after it
84+
* and `<?=` is always a tag. A bare `<?` counts only when whitespace follows,
85+
* so an `<?xml ...?>` prolog in markup stays markup. */
86+
static int php_open_tag_len(const char *s, int len, int i) {
87+
if (!php_at(s, len, i, "<?")) {
88+
return 0;
89+
}
90+
int after = i + PHP_TAG_SHORT_LEN;
91+
if (php_ieq(s, len, after, "php")) {
92+
int end = i + PHP_TAG_OPEN_LEN;
93+
return (end == len || php_is_space((unsigned char)s[end])) ? PHP_TAG_OPEN_LEN : 0;
94+
}
95+
if (after < len && s[after] == '=') {
96+
return PHP_TAG_ECHO_LEN;
97+
}
98+
if (after < len && php_is_space((unsigned char)s[after])) {
99+
return PHP_TAG_SHORT_LEN;
100+
}
101+
return 0;
102+
}
103+
104+
static bool php_has_close_tag(const char *s, int len) {
105+
for (int i = 0; i < len; i++) {
106+
if (php_at(s, len, i, "?>")) {
107+
return true;
108+
}
109+
}
110+
return false;
111+
}
112+
113+
/* Blank [from, to) in place, keeping line breaks. */
114+
static void php_blank(char *out, int from, int to) {
115+
for (int k = from; k < to; k++) {
116+
if (out[k] != '\n' && out[k] != '\r') {
117+
out[k] = ' ';
118+
}
119+
}
120+
}
121+
122+
/* Index just past the quoted string opened at i. */
123+
static int php_skip_quoted(const char *s, int len, int i) {
124+
char quote = s[i];
125+
i += SKIP_ONE;
126+
while (i < len && s[i] != quote) {
127+
i += (s[i] == '\\') ? PAIR_LEN : SKIP_ONE;
128+
}
129+
return i < len ? i + SKIP_ONE : len;
130+
}
131+
132+
/* Index just past the block comment opened at i. */
133+
static int php_skip_block_comment(const char *s, int len, int i) {
134+
for (int k = i + PAIR_LEN; k < len; k++) {
135+
if (php_at(s, len, k, "*/")) {
136+
return k + PAIR_LEN;
137+
}
138+
}
139+
return len;
140+
}
141+
142+
/* Index of the line break or `?>` that ends the one-line comment at i. */
143+
static int php_skip_line_comment(const char *s, int len, int i) {
144+
while (i < len && s[i] != '\n' && !php_at(s, len, i, "?>")) {
145+
i++;
146+
}
147+
return i;
148+
}
149+
150+
/* Parse the heredoc/nowdoc opener `<<<ID`, `<<<"ID"` or `<<<'ID'` at i. On
151+
* success stores the label span and returns the index of the line break that
152+
* ends the opener; returns PHP_NOT_FOUND when the bytes are not an opener. */
153+
static int php_heredoc_opener(const char *s, int len, int i, int *label, int *label_len) {
154+
int p = i + PHP_HEREDOC_LEN;
155+
while (p < len && php_is_blank(s[p])) {
156+
p++;
157+
}
158+
char quote = 0;
159+
if (p < len && (s[p] == '\'' || s[p] == '"')) {
160+
quote = s[p];
161+
p++;
162+
}
163+
int start = p;
164+
while (p < len && php_is_ident_char((unsigned char)s[p])) {
165+
p++;
166+
}
167+
if (p == start || !php_is_ident_start((unsigned char)s[start])) {
168+
return PHP_NOT_FOUND;
169+
}
170+
*label = start;
171+
*label_len = p - start;
172+
if (quote) {
173+
if (p >= len || s[p] != quote) {
174+
return PHP_NOT_FOUND;
175+
}
176+
p++;
177+
}
178+
if (p < len && s[p] == '\r') {
179+
p++;
180+
}
181+
return (p < len && s[p] == '\n') ? p : PHP_NOT_FOUND;
182+
}
183+
184+
/* True when the line starting at p closes the heredoc. PHP 7.3+ allows the
185+
* closing label to be indented and followed by any non-identifier byte. */
186+
static bool php_heredoc_closes(const char *s, int len, int p, int label, int label_len, int *end) {
187+
while (p < len && php_is_blank(s[p])) {
188+
p++;
189+
}
190+
if (len - p < label_len || memcmp(s + p, s + label, (size_t)label_len) != 0) {
191+
return false;
192+
}
193+
int after = p + label_len;
194+
if (after < len && php_is_ident_char((unsigned char)s[after])) {
195+
return false;
196+
}
197+
*end = after;
198+
return true;
199+
}
200+
201+
/* Index just past the heredoc/nowdoc opened by the `<<<` at i. */
202+
static int php_skip_heredoc(const char *s, int len, int i) {
203+
int label = 0;
204+
int label_len = 0;
205+
int nl = php_heredoc_opener(s, len, i, &label, &label_len);
206+
if (nl < 0) {
207+
return i + PHP_HEREDOC_LEN;
208+
}
209+
while (nl < len) {
210+
int line = nl + SKIP_ONE;
211+
int end = 0;
212+
if (php_heredoc_closes(s, len, line, label, label_len, &end)) {
213+
return end;
214+
}
215+
const char *next = memchr(s + line, '\n', (size_t)(len - line));
216+
if (!next) {
217+
return len;
218+
}
219+
nl = (int)(next - s);
220+
}
221+
return len;
222+
}
223+
224+
/* Index just past the identifier at i; sets *halt when it is __halt_compiler
225+
* (a keyword, so not a `$variable` of that name). */
226+
static int php_skip_ident(const char *s, int len, int i, bool *halt) {
227+
int start = i;
228+
while (i < len && php_is_ident_char((unsigned char)s[i])) {
229+
i++;
230+
}
231+
bool is_var = start > 0 && s[start - SKIP_ONE] == '$';
232+
*halt =
233+
!is_var && i - start == PHP_HALT_COMPILER_LEN && php_ieq(s, len, start, PHP_HALT_COMPILER);
234+
return i;
235+
}
236+
237+
/* Scan PHP code from i to the next close tag. Returns the index of that `?>`,
238+
* len when the file ends in PHP mode, or PHP_NOT_FOUND at __halt_compiler. */
239+
static int php_scan_code(const char *s, int len, int i) {
240+
while (i < len) {
241+
unsigned char c = (unsigned char)s[i];
242+
if (php_at(s, len, i, "?>")) {
243+
return i;
244+
}
245+
if (c == '\'' || c == '"' || c == '`') {
246+
i = php_skip_quoted(s, len, i);
247+
} else if (php_at(s, len, i, "/*")) {
248+
i = php_skip_block_comment(s, len, i);
249+
} else if (php_at(s, len, i, "//") || (c == '#' && !php_at(s, len, i, "#["))) {
250+
i = php_skip_line_comment(s, len, i);
251+
} else if (php_at(s, len, i, "<<<")) {
252+
i = php_skip_heredoc(s, len, i);
253+
} else if (php_is_ident_start(c)) {
254+
bool halt = false;
255+
i = php_skip_ident(s, len, i, &halt);
256+
if (halt) {
257+
return PHP_NOT_FOUND;
258+
}
259+
} else {
260+
i++;
261+
}
262+
}
263+
return len;
264+
}
265+
266+
const char *cbm_php_mask_inline_html(CBMArena *arena, const char *source, int source_len) {
267+
if (!arena || !source || source_len <= 0) {
268+
return source;
269+
}
270+
/* Fast path: a file that opens PHP at its first byte and never writes a
271+
* close tag is exactly what the grammar already accepts. */
272+
if (php_open_tag_len(source, source_len, 0) > 0 && !php_has_close_tag(source, source_len)) {
273+
return source;
274+
}
275+
276+
char *out = (char *)cbm_arena_alloc(arena, (size_t)source_len + SKIP_ONE);
277+
if (!out) {
278+
return source;
279+
}
280+
memcpy(out, source, (size_t)source_len);
281+
out[source_len] = '\0';
282+
283+
bool changed = false;
284+
bool seen_open = false;
285+
int i = 0;
286+
while (i < source_len) {
287+
/* Inline-HTML mode: blank up to the next open tag. */
288+
int html_start = i;
289+
int tag = 0;
290+
while (i < source_len && (tag = php_open_tag_len(source, source_len, i)) == 0) {
291+
i++;
292+
}
293+
if (i > html_start) {
294+
php_blank(out, html_start, i);
295+
changed = true;
296+
}
297+
if (i >= source_len) {
298+
break;
299+
}
300+
if (seen_open) {
301+
php_blank(out, i, i + tag);
302+
changed = true;
303+
}
304+
seen_open = true;
305+
306+
/* PHP mode, up to the close tag. */
307+
int close = php_scan_code(source, source_len, i + tag);
308+
if (close < 0 || close >= source_len) {
309+
break;
310+
}
311+
out[close] = ';';
312+
out[close + SKIP_ONE] = ' ';
313+
changed = true;
314+
i = close + PAIR_LEN;
315+
}
316+
return changed ? out : source;
317+
}

0 commit comments

Comments
 (0)