Skip to content

Commit ec47e40

Browse files
authored
Merge pull request DeusData#2340 from DeusData/fix/issue-2078
fix(extraction): parse every file as if its last line were terminated (DeusData#2078)
2 parents 60f6253 + ea2e36a commit ec47e40

12 files changed

Lines changed: 379 additions & 42 deletions

File tree

‎internal/cbm/cbm.c‎

Lines changed: 74 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -279,20 +279,80 @@ void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch) {
279279
typedef struct {
280280
const char *string;
281281
uint32_t length;
282+
/* #2078: serve one "\n" at offset `length` -- see cbm_parse_source. */
283+
bool virtual_newline;
282284
} CBMStringInput;
283285

284286
static const char *cbm_string_read(void *payload, uint32_t byte, TSPoint point,
285287
uint32_t *bytes_read) {
286288
(void)point;
287289
CBMStringInput *self = (CBMStringInput *)payload;
288290
if (byte >= self->length) {
291+
if (self->virtual_newline && byte == self->length) {
292+
*bytes_read = 1;
293+
return "\n";
294+
}
289295
*bytes_read = 0;
290296
return "";
291297
}
292298
*bytes_read = self->length - byte;
293299
return self->string + byte;
294300
}
295301

302+
/* Where the parser stands after reading all `len` real bytes. */
303+
static TSPoint cbm_source_end_point(const char *source, uint32_t len) {
304+
TSPoint end = {0, 0};
305+
const char *p = source;
306+
const char *stop = source + len;
307+
const char *nl;
308+
while (p < stop && (nl = memchr(p, '\n', (size_t)(stop - p))) != NULL) {
309+
end.row++;
310+
p = nl + 1;
311+
}
312+
end.column = (uint32_t)(stop - p);
313+
return end;
314+
}
315+
316+
/* #2078: parse every file as if its last line were terminated.
317+
*
318+
* Many grammars treat the line terminator as part of the construct it ends --
319+
* a markdown fence, heading or list marker, a Makefile recipe, a Dockerfile
320+
* instruction. A file whose last byte is not "\n" leaves that construct
321+
* unterminated: at best a zero-width MISSING token (#1610/#1746 excuse those at
322+
* EOF), at worst a WIDTH-BEARING error that costs the whole last line or, for a
323+
* one-line file, the whole file, and sometimes a silent loss (an unterminated
324+
* markdown heading produced no Section and no flag). The same bytes plus one
325+
* "\n" parse clean, so the absent newline is supplied instead of excused.
326+
*
327+
* The parser reads one virtual "\n" past EOF; afterwards a single tree edit
328+
* deletes it again. tree-sitter's own edit arithmetic then clamps every node
329+
* that reached into the virtual byte back to the real end -- byte offsets AND
330+
* row/column points -- so every consumer (def line ranges, node text, call
331+
* byte spans, parse-coverage ranges, the retained tree the LSP passes reuse)
332+
* sees only real bytes and real lines; nothing downstream needs to know. The
333+
* caller's buffer is never copied or written.
334+
*
335+
* A file that is empty or already ends in "\n" is parsed exactly as before. */
336+
TSTree *cbm_parse_source(TSParser *parser, const char *source, uint32_t source_len,
337+
TSParseOptions opts) {
338+
CBMStringInput input = {source, source_len, source_len > 0 && source[source_len - 1] != '\n'};
339+
TSInput ts_input = {&input, cbm_string_read, TSInputEncodingUTF8, NULL};
340+
TSTree *tree = ts_parser_parse_with_options(parser, NULL, ts_input, opts);
341+
if (tree && input.virtual_newline) {
342+
TSPoint end = cbm_source_end_point(source, source_len);
343+
TSInputEdit drop_virtual_newline = {
344+
.start_byte = source_len,
345+
.old_end_byte = source_len + 1,
346+
.new_end_byte = source_len,
347+
.start_point = end,
348+
.old_end_point = {end.row + 1, 0},
349+
.new_end_point = end,
350+
};
351+
ts_tree_edit(tree, &drop_virtual_newline);
352+
}
353+
return tree;
354+
}
355+
296356
// --- Parse timeout callback ---
297357

298358
/* Budget for the tree-sitter progress callback. The PRIMARY gate is per-thread
@@ -1262,7 +1322,11 @@ static void cbm_error_regions_push(cbm_error_regions_t *acc, TSNode n) {
12621322
* #1746: the Dockerfile grammar places that zero-width missing newline before
12631323
* trailing whitespace rather than at raw EOF. Preserve the broad exact-EOF
12641324
* rule above; only extend it past blanks when the missing token is specifically
1265-
* a newline. */
1325+
* a newline.
1326+
*
1327+
* #2078: cbm_parse_source now supplies the absent final newline, so a MISSING
1328+
* newline at EOF no longer arises from it; the rule stays for the other
1329+
* zero-width terminators a grammar can leave at EOF. */
12661330
static bool cbm_is_blank_not_newline(char c) {
12671331
return c == ' ' || c == '\t' || c == '\v' || c == '\f' || c == '\r';
12681332
}
@@ -2206,8 +2270,6 @@ static void extract_cpp_branch_views(const CBMExtractCtx *raw, const TSLanguage
22062270
return;
22072271
}
22082272
ts_parser_reset(parser);
2209-
CBMStringInput input = {view, (uint32_t)raw->source_len};
2210-
TSInput ts_input = {&input, cbm_string_read, TSInputEncodingUTF8, NULL};
22112273
TSParseOptions opts = {0};
22122274
CBMParseBudget budget = {0}; // cppcheck-suppress unreadVariable
22132275
if (timeout_micros > 0) {
@@ -2217,7 +2279,9 @@ static void extract_cpp_branch_views(const CBMExtractCtx *raw, const TSLanguage
22172279
opts.payload = &budget;
22182280
opts.progress_callback = cbm_timeout_cb;
22192281
}
2220-
TSTree *tree = ts_parser_parse_with_options(parser, NULL, ts_input, opts);
2282+
/* A view keeps the raw source's length and line structure, so it is
2283+
* parsed under the same terminated-last-line rule (#2078). */
2284+
TSTree *tree = cbm_parse_source(parser, view, (uint32_t)raw->source_len, opts);
22212285
if (!tree) {
22222286
return;
22232287
}
@@ -2325,15 +2389,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
23252389

23262390
uint64_t t0 = now_ns();
23272391

2328-
// Build string input + timeout options for parse_with_options
2329-
CBMStringInput str_input = {source, (uint32_t)source_len};
2330-
TSInput ts_input = {
2331-
&str_input,
2332-
cbm_string_read,
2333-
TSInputEncodingUTF8,
2334-
NULL,
2335-
};
2336-
2392+
// Timeout options for parse_with_options
23372393
TSParseOptions opts = {0};
23382394
CBMParseBudget budget = {0}; // cppcheck-suppress unreadVariable
23392395
uint64_t budget_ns = 0;
@@ -2361,12 +2417,14 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
23612417
/* #1735: a SQL data dump's literal-only INSERT rows carry no graph content
23622418
* but dominate its parse. Keep them out through included ranges; offsets
23632419
* and positions of everything kept are unchanged. The parser is
2364-
* thread-local and reused, so the ranges are cleared right after. */
2420+
* thread-local and reused, so the ranges are cleared right after. The last
2421+
* range ends at the real EOF, so the virtual final newline cbm_parse_source
2422+
* supplies (#2078) stays outside a ranged parse. */
23652423
CBMSqlKeptRanges sql_kept = {NULL, 0};
23662424
bool sql_ranged = language == CBM_LANG_SQL && cbm_sql_values_exclusion_on(rel_path) &&
23672425
cbm_sql_values_kept_ranges(source, (uint32_t)source_len, &sql_kept) &&
23682426
ts_parser_set_included_ranges(parser, sql_kept.items, sql_kept.count);
2369-
TSTree *tree = ts_parser_parse_with_options(parser, NULL, ts_input, opts);
2427+
TSTree *tree = cbm_parse_source(parser, source, (uint32_t)source_len, opts);
23702428
if (sql_ranged) {
23712429
(void)ts_parser_set_included_ranges(parser, NULL, 0);
23722430
}
@@ -2587,7 +2645,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
25872645
TSParser *pp_parser = get_thread_parser(ts_lang, language);
25882646
if (pp_parser) {
25892647
ts_parser_reset(pp_parser);
2590-
CBMStringInput pp_input = {expanded, (uint32_t)expanded_len};
2648+
CBMStringInput pp_input = {expanded, (uint32_t)expanded_len, false};
25912649
TSInput pp_ts_input = {
25922650
&pp_input,
25932651
cbm_string_read,

‎internal/cbm/cbm.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -894,6 +894,14 @@ void cbm_work_arena_keep_begin(void);
894894
/* Free the compaction scratch this thread kept (cbm_work_arena_release calls it). */
895895
void cbm_result_compact_release_thread(void);
896896

897+
/* Parse one whole file as if its last line ended with "\n" (#2078). The
898+
* parser sees the source plus one virtual newline when the last byte is not
899+
* already one; the returned tree is then clamped back to `source_len`, so no
900+
* node range, point or text reaches past the real bytes. Every whole-file parse
901+
* goes through here, so a retained tree and a fallback re-parse agree. */
902+
TSTree *cbm_parse_source(TSParser *parser, const char *source, uint32_t source_len,
903+
TSParseOptions opts);
904+
897905
// Extract all data from one file. Caller must call cbm_free_result().
898906
// source must remain valid for the duration of the call.
899907
// timeout_micros: per-file tree-sitter parse budget in microseconds of the

‎internal/cbm/lsp/c_lsp.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6753,7 +6753,7 @@ bool cbm_run_c_lsp_cross_with_registry_with_test_owners(CBMArena *arena, const c
67536753
return false;
67546754
const TSLanguage *ts_lang = cpp_mode ? tree_sitter_cpp() : tree_sitter_c();
67556755
ts_parser_set_language(parser, ts_lang);
6756-
tree = ts_parser_parse_string(parser, NULL, source, source_len);
6756+
tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0});
67576757
ts_parser_delete(parser);
67586758
owns_tree = true;
67596759
if (!tree)
@@ -6823,7 +6823,7 @@ bool cbm_run_c_lsp_cross_with_test_owners(CBMArena *arena, const char *source, i
68236823
return false;
68246824
const TSLanguage *ts_lang = cpp_mode ? tree_sitter_cpp() : tree_sitter_c();
68256825
ts_parser_set_language(parser, ts_lang);
6826-
tree = ts_parser_parse_string(parser, NULL, source, source_len);
6826+
tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0});
68276827
ts_parser_delete(parser);
68286828
owns_tree = true;
68296829
if (!tree)

‎internal/cbm/lsp/cs_lsp.c‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3763,8 +3763,9 @@ void cbm_run_cs_lsp_cross_with_registry(CBMArena *arena, const char *source, int
37633763
if (!parser)
37643764
return;
37653765
ts_parser_set_language(parser, tree_sitter_c_sharp());
3766-
tree = ts_parser_parse_string(
3767-
parser, NULL, source, source_len > 0 ? (uint32_t)source_len : (uint32_t)strlen(source));
3766+
tree = cbm_parse_source(parser, source,
3767+
source_len > 0 ? (uint32_t)source_len : (uint32_t)strlen(source),
3768+
(TSParseOptions){0});
37683769
ts_parser_delete(parser);
37693770
owns = true;
37703771
}
@@ -3805,8 +3806,9 @@ void cbm_run_cs_lsp_cross(CBMArena *arena, const char *source, int source_len,
38053806
if (!parser)
38063807
return;
38073808
ts_parser_set_language(parser, tree_sitter_c_sharp());
3808-
tree = ts_parser_parse_string(
3809-
parser, NULL, source, source_len > 0 ? (uint32_t)source_len : (uint32_t)strlen(source));
3809+
tree = cbm_parse_source(parser, source,
3810+
source_len > 0 ? (uint32_t)source_len : (uint32_t)strlen(source),
3811+
(TSParseOptions){0});
38103812
ts_parser_delete(parser);
38113813
owns = true;
38123814
}

‎internal/cbm/lsp/go_lsp.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3224,7 +3224,7 @@ void cbm_run_go_lsp_cross(CBMArena *arena, const char *source, int source_len,
32243224
if (!parser)
32253225
return;
32263226
ts_parser_set_language(parser, tree_sitter_go());
3227-
tree = ts_parser_parse_string(parser, NULL, source, source_len);
3227+
tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0});
32283228
owns_tree = true;
32293229
if (!tree) {
32303230
ts_parser_delete(parser);
@@ -3630,7 +3630,7 @@ void cbm_run_go_lsp_cross_with_registry(CBMArena *arena, const char *source, int
36303630
if (!parser)
36313631
return;
36323632
ts_parser_set_language(parser, tree_sitter_go());
3633-
tree = ts_parser_parse_string(parser, NULL, source, source_len);
3633+
tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0});
36343634
owns_tree = true;
36353635
if (!tree) {
36363636
ts_parser_delete(parser);

‎internal/cbm/lsp/java_lsp.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3931,7 +3931,7 @@ void cbm_run_java_lsp_cross_with_registry(CBMArena *arena, CBMFileResult *result
39313931
return;
39323932
}
39333933
ts_parser_set_language(parser, tree_sitter_java());
3934-
tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len);
3934+
tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0});
39353935
ts_parser_delete(parser);
39363936
owns_tree = true;
39373937
}
@@ -3985,7 +3985,7 @@ void cbm_run_java_lsp_cross(CBMArena *arena, const char *source, int source_len,
39853985
if (!tree) {
39863986
TSParser *parser = ts_parser_new();
39873987
ts_parser_set_language(parser, tree_sitter_java());
3988-
tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len);
3988+
tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0});
39893989
ts_parser_delete(parser);
39903990
owns_tree = true;
39913991
}

‎internal/cbm/lsp/kotlin_lsp.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5415,7 +5415,7 @@ void cbm_run_kotlin_lsp_cross(CBMArena *arena, const char *source, int source_le
54155415
return;
54165416
}
54175417
ts_parser_set_language(parser, tree_sitter_kotlin());
5418-
tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len);
5418+
tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0});
54195419
ts_parser_delete(parser);
54205420
owns_tree = true;
54215421
}

‎internal/cbm/lsp/php_lsp.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4498,7 +4498,7 @@ void cbm_run_php_lsp_cross(CBMArena *arena, const char *source, int source_len,
44984498
if (!parser)
44994499
return;
45004500
ts_parser_set_language(parser, tree_sitter_php_only());
4501-
tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len);
4501+
tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0});
45024502
owns_tree = true;
45034503
if (!tree) {
45044504
ts_parser_delete(parser);

‎internal/cbm/lsp/py_lsp.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5110,7 +5110,7 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len,
51105110
if (!parser)
51115111
return;
51125112
ts_parser_set_language(parser, tree_sitter_python());
5113-
tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len);
5113+
tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0});
51145114
owns_tree = true;
51155115
if (!tree) {
51165116
ts_parser_delete(parser);
@@ -5191,7 +5191,7 @@ void cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int
51915191
if (!parser)
51925192
return;
51935193
ts_parser_set_language(parser, tree_sitter_python());
5194-
tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len);
5194+
tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0});
51955195
owns_tree = true;
51965196
if (!tree) {
51975197
ts_parser_delete(parser);

‎internal/cbm/lsp/rust_lsp.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6377,7 +6377,7 @@ void cbm_run_rust_lsp_cross_with_registry(CBMArena *arena, const char *source, i
63776377
if (!parser)
63786378
return;
63796379
ts_parser_set_language(parser, tree_sitter_rust());
6380-
tree = ts_parser_parse_string(parser, NULL, source, source_len);
6380+
tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0});
63816381
owns_tree = true;
63826382
if (!tree) {
63836383
ts_parser_delete(parser);
@@ -6412,7 +6412,7 @@ void cbm_run_rust_lsp_cross_with_manifest(CBMArena *arena, const char *source, i
64126412
if (!parser)
64136413
return;
64146414
ts_parser_set_language(parser, tree_sitter_rust());
6415-
tree = ts_parser_parse_string(parser, NULL, source, source_len);
6415+
tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0});
64166416
owns_tree = true;
64176417
if (!tree) {
64186418
ts_parser_delete(parser);

0 commit comments

Comments
 (0)