Skip to content

Commit db2f104

Browse files
DeusDataDeusData
authored
Merge pull request #2013 from shafty023/fix/1997-extract-scratch-arena
fix(extract): AST traversal stacks no longer outlive the file they walk
2 parents 0e9ed75 + 40f2722 commit db2f104

12 files changed

Lines changed: 359 additions & 186 deletions

‎internal/cbm/cbm.c‎

Lines changed: 47 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
/* Full declaration set for the same CBMArena, and it must precede cbm.h:
2+
* internal/cbm/arena.h declares a subset and the two share the CBM_ARENA_H
3+
* guard, so whichever is included first is the one this file sees. */
4+
#include "foundation/arena.h" // cbm_arena_init_sized
15
#include "cbm.h"
26
#include "arena.h" // CBMArena, cbm_arena_init/alloc/strdup/destroy
37
#include "helpers.h"
@@ -1186,11 +1190,27 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage
11861190
return r;
11871191
}
11881192

1189-
CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLanguage language,
1190-
const char *project, const char *rel_path,
1191-
int64_t timeout_micros, const char **extra_defines,
1192-
const char **include_paths, const CBMMacroTable *macro_table,
1193-
const CBMReturnTypeTable *return_type_table) {
1193+
/* Initial block for the per-file traversal scratch arena, chosen by measuring
1194+
* arena_grow on a 14k-file TypeScript tree: it fires on one file in 12,000 at
1195+
* both this size and at 1 MB, and on most files at 256 KB, where the two
1196+
* channel walks alone are exactly 262144 bytes. 512 KB therefore buys the same
1197+
* growth behaviour as 1 MB for half the resident block per worker. It is also
1198+
* exactly MI_LARGE_MAX_OBJ_SIZE in the vendored mimalloc
1199+
* (vendored/mimalloc/include/mimalloc/types.h:426, MI_LARGE_PAGE_SIZE/8 with
1200+
* MI_ENABLE_LARGE_PAGES defaulting to 1 at :115 and not overridden here), so
1201+
* the block is still bin-allocated from a large page. Growth is not free at
1202+
* this size for the same reason: arena_grow doubles to 1 MiB, which is above
1203+
* that bound and so a singleton OS allocation. One file in twelve thousand
1204+
* pays it, which is why the cost is accepted. */
1205+
enum { CBM_EXTRACT_SCRATCH_BLOCK = CBM_SZ_512 * CBM_SZ_1K };
1206+
1207+
static CBMFileResult *extract_file_ex_body(const char *source, int source_len, CBMLanguage language,
1208+
const char *project, const char *rel_path,
1209+
int64_t timeout_micros, const char **extra_defines,
1210+
const char **include_paths,
1211+
const CBMMacroTable *macro_table,
1212+
const CBMReturnTypeTable *return_type_table,
1213+
CBMArena *scratch) {
11941214
// Allocate result on heap (arena inside for all string data)
11951215
enum { SINGLE = 1 };
11961216
CBMFileResult *result = (CBMFileResult *)calloc(SINGLE, sizeof(CBMFileResult));
@@ -1301,6 +1321,7 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua
13011321
// Build extraction context
13021322
CBMExtractCtx ctx = {
13031323
.arena = a,
1324+
.scratch = scratch,
13041325
.result = result,
13051326
.source = source,
13061327
.source_len = source_len,
@@ -1429,6 +1450,7 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua
14291450
// Build context for expanded source — extract only calls via unified extractor
14301451
CBMExtractCtx pp_ctx = {
14311452
.arena = a,
1453+
.scratch = scratch,
14321454
.result = result,
14331455
.source = expanded,
14341456
.source_len = expanded_len,
@@ -1665,6 +1687,26 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua
16651687
return result;
16661688
}
16671689

1690+
/* Public entry. Owns the traversal scratch arena for the whole of one file's
1691+
* extraction: created here, handed to the body as ctx->scratch, destroyed on
1692+
* the way out. The body has seven early returns, so bracketing it in a wrapper
1693+
* is what keeps that to one create and one destroy. If the arena cannot be
1694+
* created, the body is handed NULL and the traversal stacks fall back to the
1695+
* result arena, which is what shipped before #1997. */
1696+
CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLanguage language,
1697+
const char *project, const char *rel_path,
1698+
int64_t timeout_micros, const char **extra_defines,
1699+
const char **include_paths, const CBMMacroTable *macro_table,
1700+
const CBMReturnTypeTable *return_type_table) {
1701+
CBMArena scratch;
1702+
cbm_arena_init_sized(&scratch, CBM_EXTRACT_SCRATCH_BLOCK);
1703+
CBMFileResult *result = extract_file_ex_body(
1704+
source, source_len, language, project, rel_path, timeout_micros, extra_defines,
1705+
include_paths, macro_table, return_type_table, scratch.nblocks > 0 ? &scratch : NULL);
1706+
cbm_arena_destroy(&scratch);
1707+
return result;
1708+
}
1709+
16681710
void cbm_free_result(CBMFileResult *result) {
16691711
if (!result) {
16701712
return;

‎internal/cbm/cbm.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -589,6 +589,13 @@ typedef struct {
589589

590590
typedef struct {
591591
CBMArena *arena;
592+
/* Scratch for AST traversal, owned by the cbm_extract_file_ex call that
593+
* built this context and destroyed when it returns. Nothing a
594+
* CBMFileResult points at may be allocated here: `arena` is the result's
595+
* own, and it outlives extraction by the whole pipeline (#1997). NULL in a
596+
* context built without one, in which case the stacks fall back to
597+
* `arena`. */
598+
CBMArena *scratch;
592599
CBMFileResult *result;
593600
const char *source;
594601
int source_len;

‎internal/cbm/extract_channels.c‎

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,8 @@ static const char *literal_from_first_child(CBMExtractCtx *ctx, TSNode node) {
101101
* const table per file is sufficient for the common Socket.IO pattern). */
102102
static void scan_string_consts_js(CBMExtractCtx *ctx, chan_const_table_t *tbl) {
103103
TSNodeStack stack;
104-
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
105-
ts_nstack_push(&stack, ctx->arena, ctx->root);
104+
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
105+
ts_nstack_push(&stack, ctx->root);
106106

107107
while (stack.count > 0 && tbl->count < CHAN_CONST_CAP) {
108108
TSNode node = ts_nstack_pop(&stack);
@@ -128,15 +128,15 @@ static void scan_string_consts_js(CBMExtractCtx *ctx, chan_const_table_t *tbl) {
128128
}
129129
}
130130

131-
ts_nstack_push_children(&stack, ctx->arena, node);
131+
ts_nstack_push_children(&stack, node);
132132
}
133133
}
134134

135135
/* Python constant resolution: NAME = "value" (assignment node). */
136136
static void scan_string_consts_python(CBMExtractCtx *ctx, chan_const_table_t *tbl) {
137137
TSNodeStack stack;
138-
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
139-
ts_nstack_push(&stack, ctx->arena, ctx->root);
138+
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
139+
ts_nstack_push(&stack, ctx->root);
140140

141141
while (stack.count > 0 && tbl->count < CHAN_CONST_CAP) {
142142
TSNode node = ts_nstack_pop(&stack);
@@ -163,7 +163,7 @@ static void scan_string_consts_python(CBMExtractCtx *ctx, chan_const_table_t *tb
163163

164164
uint32_t count = ts_node_child_count(node);
165165
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
166-
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
166+
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
167167
}
168168
}
169169
}
@@ -373,15 +373,15 @@ static void extract_channels_js(CBMExtractCtx *ctx) {
373373

374374
/* Second pass: walk the tree looking for call_expression nodes. */
375375
TSNodeStack stack;
376-
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
377-
ts_nstack_push(&stack, ctx->arena, ctx->root);
376+
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
377+
ts_nstack_push(&stack, ctx->root);
378378

379379
while (stack.count > 0) {
380380
TSNode node = ts_nstack_pop(&stack);
381381
if (strcmp(ts_node_type(node), "call_expression") == 0) {
382382
js_process_call(ctx, node, &consts);
383383
}
384-
ts_nstack_push_children(&stack, ctx->arena, node);
384+
ts_nstack_push_children(&stack, node);
385385
}
386386
}
387387

@@ -549,8 +549,8 @@ static void extract_channels_python(CBMExtractCtx *ctx) {
549549
scan_string_consts_python(ctx, &consts);
550550

551551
TSNodeStack stack;
552-
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
553-
ts_nstack_push(&stack, ctx->arena, ctx->root);
552+
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
553+
ts_nstack_push(&stack, ctx->root);
554554

555555
while (stack.count > 0) {
556556
TSNode node = ts_nstack_pop(&stack);
@@ -562,7 +562,7 @@ static void extract_channels_python(CBMExtractCtx *ctx) {
562562
}
563563
uint32_t count = ts_node_child_count(node);
564564
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
565-
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
565+
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
566566
}
567567
}
568568
}
@@ -627,8 +627,8 @@ static void go_process_call(CBMExtractCtx *ctx, TSNode call) {
627627

628628
static void extract_channels_go(CBMExtractCtx *ctx) {
629629
TSNodeStack stack;
630-
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
631-
ts_nstack_push(&stack, ctx->arena, ctx->root);
630+
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
631+
ts_nstack_push(&stack, ctx->root);
632632

633633
while (stack.count > 0) {
634634
TSNode node = ts_nstack_pop(&stack);
@@ -637,7 +637,7 @@ static void extract_channels_go(CBMExtractCtx *ctx) {
637637
}
638638
uint32_t count = ts_node_child_count(node);
639639
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
640-
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
640+
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
641641
}
642642
}
643643
}
@@ -724,8 +724,8 @@ static void java_process_annotation(CBMExtractCtx *ctx, TSNode annotation) {
724724

725725
static void extract_channels_java(CBMExtractCtx *ctx) {
726726
TSNodeStack stack;
727-
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
728-
ts_nstack_push(&stack, ctx->arena, ctx->root);
727+
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
728+
ts_nstack_push(&stack, ctx->root);
729729

730730
while (stack.count > 0) {
731731
TSNode node = ts_nstack_pop(&stack);
@@ -737,7 +737,7 @@ static void extract_channels_java(CBMExtractCtx *ctx) {
737737
}
738738
uint32_t count = ts_node_child_count(node);
739739
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
740-
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
740+
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
741741
}
742742
}
743743
}
@@ -793,8 +793,8 @@ static void csharp_process_call(CBMExtractCtx *ctx, TSNode call) {
793793

794794
static void extract_channels_csharp(CBMExtractCtx *ctx) {
795795
TSNodeStack stack;
796-
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
797-
ts_nstack_push(&stack, ctx->arena, ctx->root);
796+
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
797+
ts_nstack_push(&stack, ctx->root);
798798

799799
while (stack.count > 0) {
800800
TSNode node = ts_nstack_pop(&stack);
@@ -803,7 +803,7 @@ static void extract_channels_csharp(CBMExtractCtx *ctx) {
803803
}
804804
uint32_t count = ts_node_child_count(node);
805805
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
806-
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
806+
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
807807
}
808808
}
809809
}
@@ -856,8 +856,8 @@ static void ruby_process_call(CBMExtractCtx *ctx, TSNode call) {
856856

857857
static void extract_channels_ruby(CBMExtractCtx *ctx) {
858858
TSNodeStack stack;
859-
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
860-
ts_nstack_push(&stack, ctx->arena, ctx->root);
859+
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
860+
ts_nstack_push(&stack, ctx->root);
861861

862862
while (stack.count > 0) {
863863
TSNode node = ts_nstack_pop(&stack);
@@ -866,7 +866,7 @@ static void extract_channels_ruby(CBMExtractCtx *ctx) {
866866
}
867867
uint32_t count = ts_node_child_count(node);
868868
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
869-
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
869+
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
870870
}
871871
}
872872
}
@@ -958,8 +958,8 @@ static void elixir_process_function_def(CBMExtractCtx *ctx, TSNode func_def) {
958958

959959
static void extract_channels_elixir(CBMExtractCtx *ctx) {
960960
TSNodeStack stack;
961-
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
962-
ts_nstack_push(&stack, ctx->arena, ctx->root);
961+
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
962+
ts_nstack_push(&stack, ctx->root);
963963

964964
while (stack.count > 0) {
965965
TSNode node = ts_nstack_pop(&stack);
@@ -971,7 +971,7 @@ static void extract_channels_elixir(CBMExtractCtx *ctx) {
971971
}
972972
uint32_t count = ts_node_child_count(node);
973973
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
974-
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
974+
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
975975
}
976976
}
977977
}
@@ -1034,8 +1034,8 @@ static void rust_process_call(CBMExtractCtx *ctx, TSNode call) {
10341034

10351035
static void extract_channels_rust(CBMExtractCtx *ctx) {
10361036
TSNodeStack stack;
1037-
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
1038-
ts_nstack_push(&stack, ctx->arena, ctx->root);
1037+
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
1038+
ts_nstack_push(&stack, ctx->root);
10391039

10401040
while (stack.count > 0) {
10411041
TSNode node = ts_nstack_pop(&stack);
@@ -1044,7 +1044,7 @@ static void extract_channels_rust(CBMExtractCtx *ctx) {
10441044
}
10451045
uint32_t count = ts_node_child_count(node);
10461046
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
1047-
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
1047+
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
10481048
}
10491049
}
10501050
}

‎internal/cbm/extract_defs.c‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5316,8 +5316,8 @@ static TSNode emit_elixir_module_class(CBMExtractCtx *ctx, TSNode cur) {
53165316
static void extract_elixir_call(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec) {
53175317
(void)spec;
53185318
TSNodeStack stack;
5319-
ts_nstack_init(&stack, ctx->arena, CBM_SZ_64);
5320-
ts_nstack_push(&stack, ctx->arena, node);
5319+
ts_nstack_init(&stack, ctx, CBM_SZ_64);
5320+
ts_nstack_push(&stack, node);
53215321

53225322
while (stack.count > 0) {
53235323
TSNode cur = ts_nstack_pop(&stack);
@@ -5345,7 +5345,7 @@ static void extract_elixir_call(CBMExtractCtx *ctx, TSNode node, const CBMLangSp
53455345
for (int di = (int)dbc - SKIP_CHAR; di >= 0; di--) {
53465346
TSNode dchild = ts_node_child(do_block, (uint32_t)di);
53475347
if (!ts_node_is_null(dchild) && strcmp(ts_node_type(dchild), "call") == 0) {
5348-
ts_nstack_push(&stack, ctx->arena, dchild);
5348+
ts_nstack_push(&stack, dchild);
53495349
}
53505350
}
53515351
}
@@ -6285,8 +6285,8 @@ static void extract_var_names(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec
62856285
// Used by YAML, TOML, INI, JSON.
62866286
static void walk_variables_iter(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec) {
62876287
TSNodeStack stack;
6288-
ts_nstack_init(&stack, ctx->arena, CBM_SZ_256);
6289-
ts_nstack_push(&stack, ctx->arena, root);
6288+
ts_nstack_init(&stack, ctx, CBM_SZ_256);
6289+
ts_nstack_push(&stack, root);
62906290

62916291
while (stack.count > 0) {
62926292
TSNode node = ts_nstack_pop(&stack);
@@ -6310,7 +6310,7 @@ static void walk_variables_iter(CBMExtractCtx *ctx, TSNode root, const CBMLangSp
63106310
strcmp(ck, "section") == 0 || strcmp(ck, "object") == 0 ||
63116311
strcmp(ck, "array") == 0 || strcmp(ck, "pair") == 0 || strcmp(ck, "element") == 0 ||
63126312
strcmp(ck, "content") == 0) {
6313-
ts_nstack_push(&stack, ctx->arena, child);
6313+
ts_nstack_push(&stack, child);
63146314
}
63156315
}
63166316
}
@@ -7144,10 +7144,10 @@ static void wd_push_children_reverse(wd_stack_t *s, TSNode node, const char *enc
71447144
// Push nested class nodes from a class body container onto the defs stack.
71457145
// Iteratively walks into wrapper nodes (field_declaration, template_declaration).
71467146
static void push_nested_class_nodes(TSNode body, const CBMLangSpec *spec, wd_stack_t *s,
7147-
const char *enclosing_qn, CBMArena *arena) {
7147+
const char *enclosing_qn, const CBMExtractCtx *ctx) {
71487148
TSNodeStack nc_stack;
7149-
ts_nstack_init(&nc_stack, arena, NESTED_CLASS_STACK_CAP);
7150-
ts_nstack_push(&nc_stack, arena, body);
7149+
ts_nstack_init(&nc_stack, ctx, NESTED_CLASS_STACK_CAP);
7150+
ts_nstack_push(&nc_stack, body);
71517151

71527152
while (nc_stack.count > 0) {
71537153
TSNode cur = ts_nstack_pop(&nc_stack);
@@ -7162,7 +7162,7 @@ static void push_nested_class_nodes(TSNode body, const CBMLangSpec *spec, wd_sta
71627162
const char *ck = ts_node_type(child);
71637163
if (strcmp(ck, "field_declaration") == 0 ||
71647164
strcmp(ck, "template_declaration") == 0 || strcmp(ck, "declaration") == 0) {
7165-
ts_nstack_push(&nc_stack, arena, child);
7165+
ts_nstack_push(&nc_stack, child);
71667166
}
71677167
}
71687168
}
@@ -7278,7 +7278,7 @@ static void extract_typescript_namespace_def(CBMExtractCtx *ctx, TSNode node,
72787278

72797279
// Push nested class children from a class body container onto the walk stack.
72807280
static void push_class_body_children(TSNode node, const CBMLangSpec *spec, wd_stack_t *s,
7281-
const char *new_enclosing, CBMArena *arena) {
7281+
const char *new_enclosing, const CBMExtractCtx *ctx) {
72827282
/* Use the same language-aware body selection as method extraction. The old
72837283
* independent spelling list omitted valid containers such as Scala's
72847284
* `template_body` and Solidity's contract body. Methods were extracted
@@ -7294,7 +7294,7 @@ static void push_class_body_children(TSNode node, const CBMLangSpec *spec, wd_st
72947294
body = find_class_member_body(node, spec->language);
72957295
}
72967296
if (!ts_node_is_null(body)) {
7297-
push_nested_class_nodes(body, spec, s, new_enclosing, arena);
7297+
push_nested_class_nodes(body, spec, s, new_enclosing, ctx);
72987298
return;
72997299
}
73007300

@@ -7880,7 +7880,7 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
78807880
if (cbm_kind_in_set(node, spec->class_node_types)) {
78817881
extract_class_def(ctx, node, spec);
78827882
const char *new_enclosing = compute_class_qn(ctx, node, frame.enclosing_class_qn);
7883-
push_class_body_children(node, spec, &s, new_enclosing, ctx->arena);
7883+
push_class_body_children(node, spec, &s, new_enclosing, ctx);
78847884
continue;
78857885
}
78867886

0 commit comments

Comments
 (0)