Skip to content

Commit 96f9100

Browse files
committed
Merge branch 'main' into fix/elixir-extraction-defects
2 parents ee9d136 + 80eb92a commit 96f9100

46 files changed

Lines changed: 3532 additions & 329 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Makefile.cbm‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,7 @@ EXTRACTION_SRCS = \
348348
$(CBM_DIR)/extract_channels.c \
349349
$(CBM_DIR)/extract_k8s.c \
350350
$(CBM_DIR)/extract_dbt.c \
351+
$(CBM_DIR)/sql_values.c \
351352
$(CBM_DIR)/helpers.c \
352353
$(CBM_DIR)/result_compact.c \
353354
$(CBM_DIR)/result_spill.c \

‎internal/cbm/cbm.c‎

Lines changed: 86 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include "lsp/kotlin_lsp.h"
2222
#include "lsp/rust_lsp.h"
2323
#include "preprocessor.h"
24+
#include "sql_values.h" // #1735: literal INSERT rows kept out of the SQL parse
2425
#include "foundation/compat.h"
2526
#include "foundation/compat_fs.h" // cbm_fopen — crash-supervisor per-file marker write
2627
#include "foundation/hash_table.h" // CBMHashTable — crash-supervisor quarantine set
@@ -1773,6 +1774,19 @@ static void cbm_subtract_recovered_regions(cbm_error_regions_t *regs, const CBMD
17731774
*regs = out;
17741775
}
17751776

1777+
#ifdef CBM_ENABLE_TEST_SEAMS
1778+
/* Source bytes this thread has read to locate lines for the #1071 macro check
1779+
* (walks and line-table builds). Lets a test pin the check's cost as a count
1780+
* instead of a clock (#1735). Compiled only into seam-enabled test artifacts. */
1781+
static CBM_TLS uint64_t tl_macro_line_scan_bytes = 0;
1782+
uint64_t cbm_test_macro_line_scan_bytes(void) {
1783+
return tl_macro_line_scan_bytes;
1784+
}
1785+
#define CBM_MACRO_LINE_SCAN(n) (tl_macro_line_scan_bytes += (uint64_t)(n))
1786+
#else
1787+
#define CBM_MACRO_LINE_SCAN(n) ((void)0)
1788+
#endif
1789+
17761790
/* #1071: a function-like macro invocation whose argument is a type token
17771791
* (e.g. ALLOC(int, n)) makes tree-sitter's C/C++ grammar emit an ERROR node — it
17781792
* parses `int` in expression position — which would be recorded as a parse_partial
@@ -1862,6 +1876,7 @@ static bool cbm_span_is_macro_invocation(const char *src, int src_len, uint32_t
18621876
}
18631877
}
18641878
if (line != start_line) {
1879+
CBM_MACRO_LINE_SCAN(span_start);
18651880
return false;
18661881
}
18671882
int span_end = span_start;
@@ -1870,6 +1885,7 @@ static bool cbm_span_is_macro_invocation(const char *src, int src_len, uint32_t
18701885
line++;
18711886
}
18721887
}
1888+
CBM_MACRO_LINE_SCAN(span_end);
18731889
return cbm_byte_span_is_macro_invocation(src, src_len, span_start, span_end, defs);
18741890
}
18751891

@@ -1896,21 +1912,59 @@ static bool cbm_region_inside_callable(uint32_t rs, uint32_t re, const CBMDefArr
18961912
return false;
18971913
}
18981914

1915+
/* cbm_span_is_macro_invocation over a cbm_line_offsets table: offs[k] is where
1916+
* 1-based line k+1 starts and offs[nlines] is the end of the source, so a span
1917+
* costs two reads instead of a walk from byte 0. Same answer as the walk; falls
1918+
* back to it when there is no table. */
1919+
static bool cbm_lines_are_macro_invocation(const char *src, int src_len, const uint32_t *offs,
1920+
uint32_t nlines, uint32_t start_line, uint32_t end_line,
1921+
const CBMDefArray *defs) {
1922+
if (!offs) {
1923+
return cbm_span_is_macro_invocation(src, src_len, start_line, end_line, defs);
1924+
}
1925+
if (!src || src_len <= 0 || !defs || start_line == 0 || end_line < start_line ||
1926+
start_line > nlines) {
1927+
return false;
1928+
}
1929+
uint32_t last = end_line < nlines ? end_line : nlines;
1930+
return cbm_byte_span_is_macro_invocation(src, src_len, (int)offs[start_line - 1],
1931+
(int)offs[last], defs);
1932+
}
1933+
1934+
/* #1071 subtraction. Both questions are pure, so asking the cheap one first
1935+
* changes no answer: "is the region inside a function?" reads only the
1936+
* definition list, while "is it a macro call?" has to find the region's lines
1937+
* in the source. A region no function encloses — every region of a file with
1938+
* no functions, such as a SQL data dump — never touches the source, and the
1939+
* ones that do share one line table built on first use. The old order walked
1940+
* the source from byte 0 for every region: regions x file bytes (#1735). */
18991941
static void cbm_subtract_macro_invocation_regions(cbm_error_regions_t *regs,
19001942
const CBMDefArray *defs, const char *src,
19011943
int src_len) {
1944+
uint32_t *offs = NULL;
1945+
uint32_t nlines = 0;
1946+
bool offs_built = false;
19021947
int kept = 0;
19031948
for (int i = 0; i < regs->count; i++) {
1904-
bool benign =
1905-
cbm_span_is_macro_invocation(src, src_len, regs->starts[i], regs->ends[i], defs) &&
1906-
cbm_region_inside_callable(regs->starts[i], regs->ends[i], defs);
1949+
uint32_t rs = regs->starts[i];
1950+
uint32_t re = regs->ends[i];
1951+
bool benign = false;
1952+
if (cbm_region_inside_callable(rs, re, defs)) {
1953+
if (!offs_built) {
1954+
offs = cbm_line_offsets(src, src_len, &nlines);
1955+
offs_built = true;
1956+
CBM_MACRO_LINE_SCAN(src_len);
1957+
}
1958+
benign = cbm_lines_are_macro_invocation(src, src_len, offs, nlines, rs, re, defs);
1959+
}
19071960
if (!benign) {
1908-
regs->starts[kept] = regs->starts[i];
1909-
regs->ends[kept] = regs->ends[i];
1961+
regs->starts[kept] = rs;
1962+
regs->ends[kept] = re;
19101963
kept++;
19111964
}
19121965
}
19131966
regs->count = kept;
1967+
cbm_free(CBM_MEM_CLASS_EXTRACT, offs);
19141968
}
19151969

19161970
/* Push [start, end] after trimming no-code lines off both ends. A run made
@@ -2072,6 +2126,21 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage
20722126
enum { CBM_EXTRACT_SCRATCH_BLOCK = CBM_SZ_512 * CBM_SZ_1K };
20732127
enum { CBM_EXTRACT_SCRATCH_KEEP_BYTES = 4 * CBM_SZ_1K * CBM_SZ_1K };
20742128

2129+
/* The #1735 value-row exclusion is always on. Test builds can turn it off for
2130+
* one file (CBM_TEST_SQL_FULL_PARSE_ON=<rel_path substring>) so a test can
2131+
* compare the excluded parse against the full one on the same source. */
2132+
static bool cbm_sql_values_exclusion_on(const char *rel_path) {
2133+
#ifdef CBM_ENABLE_TEST_SEAMS
2134+
const char *full_on = getenv("CBM_TEST_SQL_FULL_PARSE_ON");
2135+
if (full_on && full_on[0] && rel_path && strstr(rel_path, full_on)) {
2136+
return false;
2137+
}
2138+
#else
2139+
(void)rel_path;
2140+
#endif
2141+
return true;
2142+
}
2143+
20752144
static CBMFileResult *extract_file_ex_body(const char *source, int source_len, CBMLanguage language,
20762145
const char *project, const char *rel_path,
20772146
int64_t timeout_micros, const char **extra_defines,
@@ -2180,7 +2249,19 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
21802249
#endif
21812250
}
21822251

2252+
/* #1735: a SQL data dump's literal-only INSERT rows carry no graph content
2253+
* but dominate its parse. Keep them out through included ranges; offsets
2254+
* and positions of everything kept are unchanged. The parser is
2255+
* thread-local and reused, so the ranges are cleared right after. */
2256+
CBMSqlKeptRanges sql_kept = {NULL, 0};
2257+
bool sql_ranged = language == CBM_LANG_SQL && cbm_sql_values_exclusion_on(rel_path) &&
2258+
cbm_sql_values_kept_ranges(source, (uint32_t)source_len, &sql_kept) &&
2259+
ts_parser_set_included_ranges(parser, sql_kept.items, sql_kept.count);
21832260
TSTree *tree = ts_parser_parse_with_options(parser, NULL, ts_input, opts);
2261+
if (sql_ranged) {
2262+
(void)ts_parser_set_included_ranges(parser, NULL, 0);
2263+
}
2264+
cbm_sql_kept_ranges_free(&sql_kept);
21842265
uint64_t t1 = now_ns();
21852266
#ifdef CBM_ENABLE_TEST_SEAMS
21862267
t1 += tl_parse_wall_seam_offset_ns; /* the stall seam inflates every wall reading */

‎internal/cbm/cbm.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -800,6 +800,12 @@ void cbm_free_tree(CBMFileResult *result);
800800
// Free a standalone TSTree pointer (for Go layer cleanup).
801801
void cbm_free_tree_ptr(TSTree *tree);
802802

803+
#ifdef CBM_ENABLE_TEST_SEAMS
804+
// Test-only: source bytes this thread has read to locate lines for the #1071
805+
// macro-invocation check, cumulative (#1735).
806+
uint64_t cbm_test_macro_line_scan_bytes(void);
807+
#endif
808+
803809
// Reset the thread-local parser's internal state, releasing slab-allocated
804810
// subtrees. Must be called BEFORE cbm_slab_reset_thread() so the slab rebuild
805811
// doesn't corrupt live parser state.

‎internal/cbm/extract_defs.c‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -87,10 +87,18 @@ static bool try_append_ident(const char *source, uint32_t s, int len, uint32_t *
8787
}
8888

8989
/* Walk AST body, collect unique identifier text as space-separated string.
90-
* Returns arena-allocated string or NULL. */
90+
* Contract: the first BT_MAX_IDENTS unique identifiers in pre-order (source
91+
* order), bounded by BT_BUF bytes. The ident/byte caps ARE the sampling
92+
* contract; the pending stack is not — BT_STACK is only the on-stack first
93+
* chunk and spills to the heap, because a full fixed stack dropped a wide
94+
* body's FIRST children and sampled a mid-body window instead.
95+
* Returns arena-allocated string, or NULL when there are no identifiers or the
96+
* stack could not be grown (logged; no truncated sample is ever returned). */
9197
static char *extract_body_ident_tokens(CBMExtractCtx *ctx, TSNode body) {
9298
enum { BT_STACK = 512, BT_BUF = 2048, BT_MAX_IDENTS = 128, BT_SEEN = 256, BT_SEEN_MASK = 255 };
93-
TSNode bt_stack[BT_STACK];
99+
TSNode bt_inline[BT_STACK];
100+
TSNode *bt_stack = bt_inline;
101+
int bt_cap = BT_STACK;
94102
int bt_top = 0;
95103
bt_stack[bt_top++] = body;
96104
char bt_buf[BT_BUF];
@@ -120,11 +128,17 @@ static char *extract_body_ident_tokens(CBMExtractCtx *ctx, TSNode body) {
120128
}
121129
}
122130
} else {
123-
for (int i = (int)nc - SKIP_ONE; i >= 0 && bt_top < BT_STACK; i--) {
131+
if (!cbm_walk_stack_reserve((void **)&bt_stack, &bt_cap, bt_top + (int)nc,
132+
sizeof(TSNode), bt_inline, "body_ident_tokens")) {
133+
bt_pos = 0; /* sentinel: no sample rather than a truncated one */
134+
break;
135+
}
136+
for (int i = (int)nc - SKIP_ONE; i >= 0; i--) {
124137
bt_stack[bt_top++] = ts_node_child(nd, (uint32_t)i);
125138
}
126139
}
127140
}
141+
cbm_walk_stack_release(bt_stack, bt_inline);
128142
if (bt_pos > 0) {
129143
bt_buf[bt_pos] = '\0';
130144
return cbm_arena_strdup(ctx->arena, bt_buf);

‎internal/cbm/extract_usages.c‎

Lines changed: 39 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -627,14 +627,38 @@ static bool is_elixir_def_binding(CBMExtractCtx *ctx, TSNode node) {
627627
return false;
628628
}
629629

630-
static bool is_first_named_part_of(TSNode node, const char *container_kind) {
631-
for (TSNode parent = ts_node_parent(node); !ts_node_is_null(parent);
632-
parent = ts_node_parent(parent)) {
630+
/* Climbs from `node` to its nearest `container_kind` ancestor and reports
631+
* whether `node` sits in that ancestor's first named child. It runs for every
632+
* named leaf of ReScript, Julia, Typst, Elm, PureScript and Nickel files, so it
633+
* climbs the unified walk's cursor (O(1) per hop) whenever the walk is on
634+
* `node`. ts_node_parent answers by descending from the ROOT and scanning the
635+
* children at every level: on an error-heavy parse (a binary Godot `.res`
636+
* resource yields a root with ~170k children per MB) each call is
637+
* O(root children), and one climb per leaf made the file quadratic (#2176:
638+
* 1 MB spent 46 s in this walk, 4.7 MB never finished). The node-based climb
639+
* remains only for callers outside the walk. */
640+
static bool is_first_named_part_of(TSNode node, const char *container_kind, WalkState *state) {
641+
TSTreeCursor *cursor = reset_occurrence_cursor(state, node);
642+
TSNode current = node;
643+
for (;;) {
644+
TSNode parent;
645+
if (cursor) {
646+
if (!ts_tree_cursor_goto_parent(cursor)) {
647+
return false;
648+
}
649+
parent = ts_tree_cursor_current_node(cursor);
650+
} else {
651+
usage_slow_parent_fallback_test_note(); /* one root descent per hop */
652+
parent = ts_node_parent(current);
653+
if (ts_node_is_null(parent)) {
654+
return false;
655+
}
656+
}
633657
if (strcmp(ts_node_type(parent), container_kind) == 0) {
634658
return named_child_contains(parent, 0, node);
635659
}
660+
current = parent;
636661
}
637-
return false;
638662
}
639663

640664
static bool is_wolfram_lhs(TSNode node) {
@@ -1037,8 +1061,8 @@ static bool is_pkl_declaration_binding(TSNode node) {
10371061
return false;
10381062
}
10391063

1040-
static bool is_policy_binding(CBMExtractCtx *ctx, TSNode node,
1041-
const CBMOccurrenceSpec *occurrence) {
1064+
static bool is_policy_binding(CBMExtractCtx *ctx, TSNode node, const CBMOccurrenceSpec *occurrence,
1065+
WalkState *state) {
10421066
switch (occurrence->policy) {
10431067
case CBM_OCCURRENCE_LISP_DEF:
10441068
return is_lisp_def_binding(ctx, node);
@@ -1049,11 +1073,11 @@ static bool is_policy_binding(CBMExtractCtx *ctx, TSNode node,
10491073
case CBM_OCCURRENCE_ELIXIR_DEF:
10501074
return is_elixir_def_binding(ctx, node);
10511075
case CBM_OCCURRENCE_JULIA_FUNCTION:
1052-
return is_first_named_part_of(node, "function_definition");
1076+
return is_first_named_part_of(node, "function_definition", state);
10531077
case CBM_OCCURRENCE_WOLFRAM_SET:
10541078
return is_wolfram_lhs(node);
10551079
case CBM_OCCURRENCE_TYPST_LET:
1056-
return is_first_named_part_of(node, "let");
1080+
return is_first_named_part_of(node, "let", state);
10571081
case CBM_OCCURRENCE_AGDA_FUNCTION:
10581082
for (TSNode parent = ts_node_parent(node); !ts_node_is_null(parent);
10591083
parent = ts_node_parent(parent)) {
@@ -1069,14 +1093,14 @@ static bool is_policy_binding(CBMExtractCtx *ctx, TSNode node,
10691093
case CBM_OCCURRENCE_HCL_ATTRIBUTE:
10701094
return is_hcl_attribute_binding(node);
10711095
case CBM_OCCURRENCE_ELM_VALUE:
1072-
return is_first_named_part_of(node, "value_declaration");
1096+
return is_first_named_part_of(node, "value_declaration", state);
10731097
case CBM_OCCURRENCE_RESCRIPT_LET:
1074-
return is_first_named_part_of(node, "let_binding");
1098+
return is_first_named_part_of(node, "let_binding", state);
10751099
case CBM_OCCURRENCE_PURESCRIPT_LHS:
1076-
return is_first_named_part_of(node, "function");
1100+
return is_first_named_part_of(node, "function", state);
10771101
case CBM_OCCURRENCE_NICKEL_LET:
1078-
return is_first_named_part_of(node, "let_binding") ||
1079-
is_first_named_part_of(node, "pattern_fun");
1102+
return is_first_named_part_of(node, "let_binding", state) ||
1103+
is_first_named_part_of(node, "pattern_fun", state);
10801104
case CBM_OCCURRENCE_ERLANG_CLAUSE:
10811105
return is_erlang_clause_binding(node);
10821106
case CBM_OCCURRENCE_NIX_FUNCTION:
@@ -1124,7 +1148,7 @@ static bool is_binding_occurrence(CBMExtractCtx *ctx, TSNode node, const CBMLang
11241148
if (is_exact_language_binding(ctx, node, state)) {
11251149
return true;
11261150
}
1127-
if (is_policy_binding(ctx, node, occurrence)) {
1151+
if (is_policy_binding(ctx, node, occurrence, state)) {
11281152
return true;
11291153
}
11301154

@@ -2634,7 +2658,7 @@ void handle_usages(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec, Wal
26342658
bool possible_binding_leaf =
26352659
ts_node_is_named(node) && ts_node_named_child_count(node) == 0 &&
26362660
(state->inside_import || is_exact_language_binding(ctx, node, state) ||
2637-
is_policy_binding(ctx, node, occurrence));
2661+
is_policy_binding(ctx, node, occurrence, state));
26382662
if (!reference_node && !possible_binding_leaf) {
26392663
return;
26402664
}

0 commit comments

Comments
 (0)