Skip to content

Commit 174e56b

Browse files
committed
fix(extract): heap-allocate walk_defs frame stack — fix overflow + >4096 truncation (DeusData#668)
walk_defs used a fixed `walk_defs_frame_t stack[4096]` — a single ~160 KB frame on the C stack. Two problems: - On small thread stacks (notably the pre-2026-03 Windows 1 MB main thread) that huge frame overflowed the stack during the definitions pass — the 0xC00000FD crash a reporter hit indexing a large SQL file (DeusData#668). Reproduced locally by running extraction under a constrained stack (~208 KB) + ASan. - The `top < 4096` push guards SILENTLY DROPPED every top-level definition past 4096, so a file with more than 4096 top-level defs was only partially indexed. Replace the fixed array with a growable heap stack (wd_stack_t + wd_push): a 256-frame initial buffer that doubles on demand, bounded by a generous, env-configurable ceiling (CBM_WALK_DEFS_MAX, default 8M frames) that WARNs once instead of dropping, and returns cleanly on realloc OOM (no NULL deref). A normal file now uses ~10 KB of heap instead of 160 KB of stack, and files with thousands of top-level defs are fully extracted. The frame stack is a pure index-addressed LIFO (frames copied by value on pop), so heap-backing is semantically identical. Reproduce-first: walk_defs_no_truncation_over_4096_issue668 — 5000 top-level defs must all be extracted (RED on the old 4096 cap, GREEN after). Full suite 5748/0, no ASan/UBSan reports on the new alloc. Root cause was pinned by reproduction (constrained-stack + ASan), refuting earlier grammar/scanner hypotheses — extraction and the vendored tree-sitter tree ops are otherwise iterative. The separate error-recovery assertion mode (symbol<token_count) was not locally reproducible and is addressed by the forthcoming general crash-containment layer. Refs DeusData#668 Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent b9b7905 commit 174e56b

2 files changed

Lines changed: 116 additions & 31 deletions

File tree

‎internal/cbm/extract_defs.c‎

Lines changed: 88 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,15 @@
33
#include "helpers.h"
44
#include "lang_specs.h"
55
#include "foundation/constants.h"
6+
#include "foundation/platform.h" // safe_realloc (frees old on failure)
7+
#include "foundation/log.h" // cbm_log_warn
68
#include "extract_node_stack.h"
79
#include "simhash/minhash.h"
810
#include "semantic/ast_profile.h"
911
#include "tree_sitter/api.h" // TSNode, ts_node_*
1012
#include <stdint.h> // uint32_t
13+
#include <stdio.h> // snprintf
14+
#include <stdlib.h> // getenv, atoi
1115
#include <string.h>
1216
#include <ctype.h>
1317

@@ -5538,12 +5542,70 @@ typedef struct {
55385542
const char *enclosing_class_qn; // saved context for class nesting
55395543
} walk_defs_frame_t;
55405544

5541-
#define CBM_WALK_DEFS_STACK_CAP 4096
5545+
/* #668: walk_defs previously used a fixed `walk_defs_frame_t stack[4096]` — a
5546+
* single ~160 KB C-stack frame. That overflowed small thread stacks (the
5547+
* pre-2026-03 Windows 1 MB main thread) on the definitions pass, and its
5548+
* `top < 4096` push guards SILENTLY DROPPED every top-level definition past
5549+
* 4096. Use a growable heap stack instead: a tiny initial footprint that doubles
5550+
* on demand, bounded by a generous, env-configurable ceiling that WARNs (once)
5551+
* rather than dropping — so a file with thousands of top-level defs is fully
5552+
* extracted, and a pathological one degrades to a warned, bounded skip instead
5553+
* of an OOM or a stack overflow. */
5554+
typedef struct {
5555+
walk_defs_frame_t *data;
5556+
int top;
5557+
int cap;
5558+
const char *path; // for the WARN when the ceiling is hit (may be NULL)
5559+
bool warned;
5560+
} wd_stack_t;
5561+
5562+
// Generous safety ceiling (frames), env-overridable via CBM_WALK_DEFS_MAX.
5563+
// Realistic files never approach this; it only bounds a pathological/adversarial
5564+
// file so extraction degrades to a warned skip rather than unbounded memory.
5565+
static int wd_stack_max(void) {
5566+
const char *e = getenv("CBM_WALK_DEFS_MAX");
5567+
if (e) {
5568+
int v = atoi(e);
5569+
if (v > 0) {
5570+
return v;
5571+
}
5572+
}
5573+
return 8 * 1024 * 1024; // 8M frames (~320 MB) default
5574+
}
5575+
5576+
static void wd_push(wd_stack_t *s, TSNode node, const char *enclosing_qn) {
5577+
if (s->top >= s->cap) {
5578+
int ncap = s->cap ? s->cap * 2 : 256;
5579+
if (ncap > wd_stack_max()) {
5580+
if (!s->warned) {
5581+
char lim[24];
5582+
snprintf(lim, sizeof(lim), "%d", wd_stack_max());
5583+
cbm_log_warn("extract.walk_defs_capped", "limit", lim, "path",
5584+
s->path ? s->path : "");
5585+
s->warned = true;
5586+
}
5587+
return; // bounded: stop growing (warned, not silent)
5588+
}
5589+
walk_defs_frame_t *nd = safe_realloc(s->data, (size_t)ncap * sizeof(walk_defs_frame_t));
5590+
if (!nd) {
5591+
/* OOM — safe_realloc already freed the old buffer. Bail cleanly: drop
5592+
* pending frames so the walk_defs loop drains and exits without a NULL
5593+
* deref; extraction keeps whatever was already emitted. */
5594+
s->data = NULL;
5595+
s->cap = 0;
5596+
s->top = 0;
5597+
return;
5598+
}
5599+
s->data = nd;
5600+
s->cap = ncap;
5601+
}
5602+
s->data[s->top++] = (walk_defs_frame_t){node, enclosing_qn};
5603+
}
55425604

55435605
// Push nested class nodes from a class body container onto the defs stack.
55445606
// Iteratively walks into wrapper nodes (field_declaration, template_declaration).
5545-
static void push_nested_class_nodes(TSNode body, const CBMLangSpec *spec, walk_defs_frame_t *stack,
5546-
int *top, const char *enclosing_qn, CBMArena *arena) {
5607+
static void push_nested_class_nodes(TSNode body, const CBMLangSpec *spec, wd_stack_t *s,
5608+
const char *enclosing_qn, CBMArena *arena) {
55475609
TSNodeStack nc_stack;
55485610
ts_nstack_init(&nc_stack, arena, NESTED_CLASS_STACK_CAP);
55495611
ts_nstack_push(&nc_stack, arena, body);
@@ -5554,9 +5616,7 @@ static void push_nested_class_nodes(TSNode body, const CBMLangSpec *spec, walk_d
55545616
for (int i = (int)nc - SKIP_CHAR; i >= 0; i--) {
55555617
TSNode child = ts_node_child(cur, (uint32_t)i);
55565618
if (cbm_kind_in_set(child, spec->class_node_types)) {
5557-
if (*top < CBM_WALK_DEFS_STACK_CAP) {
5558-
stack[(*top)++] = (walk_defs_frame_t){child, enclosing_qn};
5559-
}
5619+
wd_push(s, child, enclosing_qn);
55605620
} else {
55615621
const char *ck = ts_node_type(child);
55625622
if (strcmp(ck, "field_declaration") == 0 ||
@@ -5624,8 +5684,8 @@ static const char *compute_class_qn(CBMExtractCtx *ctx, TSNode node, const char
56245684
}
56255685

56265686
// Push nested class children from a class body container onto the walk stack.
5627-
static void push_class_body_children(TSNode node, const CBMLangSpec *spec, walk_defs_frame_t *stack,
5628-
int *top, const char *new_enclosing, CBMArena *arena) {
5687+
static void push_class_body_children(TSNode node, const CBMLangSpec *spec, wd_stack_t *s,
5688+
const char *new_enclosing, CBMArena *arena) {
56295689
uint32_t nc = ts_node_child_count(node);
56305690
for (uint32_t ci = 0; ci < nc; ci++) {
56315691
TSNode child = ts_node_child(node, ci);
@@ -5638,13 +5698,13 @@ static void push_class_body_children(TSNode node, const CBMLangSpec *spec, walk_
56385698
// double-extracted) as top-level functions. Gated to Groovy so other
56395699
// grammars that also name a node "closure" are unaffected.
56405700
(strcmp(ck, "closure") == 0 && spec->language == CBM_LANG_GROOVY)) {
5641-
push_nested_class_nodes(child, spec, stack, top, new_enclosing, arena);
5701+
push_nested_class_nodes(child, spec, s, new_enclosing, arena);
56425702
return;
56435703
}
56445704
}
56455705
// No body found — push all children directly
5646-
for (int ci = (int)nc - SKIP_CHAR; ci >= 0 && *top < CBM_WALK_DEFS_STACK_CAP; ci--) {
5647-
stack[(*top)++] = (walk_defs_frame_t){ts_node_child(node, (uint32_t)ci), new_enclosing};
5706+
for (int ci = (int)nc - SKIP_CHAR; ci >= 0; ci--) {
5707+
wd_push(s, ts_node_child(node, (uint32_t)ci), new_enclosing);
56485708
}
56495709
}
56505710

@@ -6030,12 +6090,12 @@ static void recover_kotlin_error_classes(CBMExtractCtx *ctx, TSNode err_node) {
60306090

60316091
static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec, int depth_unused) {
60326092
(void)depth_unused;
6033-
walk_defs_frame_t stack[CBM_WALK_DEFS_STACK_CAP];
6034-
int top = 0;
6035-
stack[top++] = (walk_defs_frame_t){root, ctx->enclosing_class_qn};
6093+
wd_stack_t s = {0};
6094+
s.path = ctx->rel_path;
6095+
wd_push(&s, root, ctx->enclosing_class_qn);
60366096

6037-
while (top > 0) {
6038-
walk_defs_frame_t frame = stack[--top];
6097+
while (s.top > 0) {
6098+
walk_defs_frame_t frame = s.data[--s.top];
60396099
TSNode node = frame.node;
60406100
ctx->enclosing_class_qn = frame.enclosing_class_qn;
60416101
const char *kind = ts_node_type(node);
@@ -6072,9 +6132,8 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
60726132
// field, double-mint). Push children so nested tags/defs are still
60736133
// traversed, then skip the generic func path.
60746134
uint32_t cc = ts_node_child_count(node);
6075-
for (int i = (int)cc - SKIP_CHAR; i >= 0 && top < CBM_WALK_DEFS_STACK_CAP; i--) {
6076-
stack[top++] =
6077-
(walk_defs_frame_t){ts_node_child(node, (uint32_t)i), frame.enclosing_class_qn};
6135+
for (int i = (int)cc - SKIP_CHAR; i >= 0; i--) {
6136+
wd_push(&s, ts_node_child(node, (uint32_t)i), frame.enclosing_class_qn);
60786137
}
60796138
continue;
60806139
}
@@ -6087,9 +6146,8 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
60876146
// still carries the quotes. Push children so nested defines are still
60886147
// traversed, then skip the generic func path.
60896148
uint32_t cc = ts_node_child_count(node);
6090-
for (int i = (int)cc - SKIP_CHAR; i >= 0 && top < CBM_WALK_DEFS_STACK_CAP; i--) {
6091-
stack[top++] =
6092-
(walk_defs_frame_t){ts_node_child(node, (uint32_t)i), frame.enclosing_class_qn};
6149+
for (int i = (int)cc - SKIP_CHAR; i >= 0; i--) {
6150+
wd_push(&s, ts_node_child(node, (uint32_t)i), frame.enclosing_class_qn);
60936151
}
60946152
continue;
60956153
}
@@ -6116,9 +6174,8 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
61166174
ts_node_is_null(
61176175
find_first_descendant_by_kind(node, "func_type", CBM_DESCENDANT_MAX_DEPTH))) {
61186176
uint32_t cc = ts_node_child_count(node);
6119-
for (int i = (int)cc - SKIP_CHAR; i >= 0 && top < CBM_WALK_DEFS_STACK_CAP; i--) {
6120-
stack[top++] =
6121-
(walk_defs_frame_t){ts_node_child(node, (uint32_t)i), frame.enclosing_class_qn};
6177+
for (int i = (int)cc - SKIP_CHAR; i >= 0; i--) {
6178+
wd_push(&s, ts_node_child(node, (uint32_t)i), frame.enclosing_class_qn);
61226179
}
61236180
continue;
61246181
}
@@ -6158,25 +6215,25 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
61586215
if (is_namespace_scope_kind(ctx->language, kind)) {
61596216
const char *new_enclosing = compute_class_qn(ctx, node, frame.enclosing_class_qn);
61606217
uint32_t nsc = ts_node_child_count(node);
6161-
for (int i = (int)nsc - SKIP_CHAR; i >= 0 && top < CBM_WALK_DEFS_STACK_CAP; i--) {
6162-
stack[top++] = (walk_defs_frame_t){ts_node_child(node, (uint32_t)i), new_enclosing};
6218+
for (int i = (int)nsc - SKIP_CHAR; i >= 0; i--) {
6219+
wd_push(&s, ts_node_child(node, (uint32_t)i), new_enclosing);
61636220
}
61646221
continue;
61656222
}
61666223

61676224
if (cbm_kind_in_set(node, spec->class_node_types)) {
61686225
extract_class_def(ctx, node, spec);
61696226
const char *new_enclosing = compute_class_qn(ctx, node, frame.enclosing_class_qn);
6170-
push_class_body_children(node, spec, stack, &top, new_enclosing, ctx->arena);
6227+
push_class_body_children(node, spec, &s, new_enclosing, ctx->arena);
61716228
continue;
61726229
}
61736230

61746231
uint32_t count = ts_node_child_count(node);
6175-
for (int i = (int)count - SKIP_CHAR; i >= 0 && top < CBM_WALK_DEFS_STACK_CAP; i--) {
6176-
stack[top++] =
6177-
(walk_defs_frame_t){ts_node_child(node, (uint32_t)i), frame.enclosing_class_qn};
6232+
for (int i = (int)count - SKIP_CHAR; i >= 0; i--) {
6233+
wd_push(&s, ts_node_child(node, (uint32_t)i), frame.enclosing_class_qn);
61786234
}
61796235
}
6236+
free(s.data);
61806237
}
61816238

61826239
void cbm_extract_definitions(CBMExtractCtx *ctx) {

‎tests/test_extraction.c‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3112,6 +3112,33 @@ TEST(extract_non_perl_method_call_not_flagged_is_method) {
31123112
PASS();
31133113
}
31143114

3115+
/* #668: walk_defs used a fixed `walk_defs_frame_t stack[4096]` — a ~160 KB
3116+
* C-stack frame that overflowed small thread stacks (the reporter's crash was in
3117+
* the "definitions pass" on a large SQL file), and whose `top < 4096` push guards
3118+
* SILENTLY DROPPED every top-level definition past 4096. The growable heap stack
3119+
* fixes both: a file with > 4096 top-level defs must extract ALL of them. RED
3120+
* before the fix (extracted count capped near 4096), GREEN after. */
3121+
TEST(walk_defs_no_truncation_over_4096_issue668) {
3122+
enum { N = 5000 };
3123+
/* N top-level Python defs → N Function defs, all direct module children, so
3124+
* walk_defs pushes all N children at once — the >4096 truncation site. */
3125+
size_t cap = (size_t)N * 24 + 1;
3126+
char *src = (char *)malloc(cap);
3127+
ASSERT_NOT_NULL(src);
3128+
size_t off = 0;
3129+
for (int i = 0; i < N; i++) {
3130+
off += (size_t)snprintf(src + off, cap - off, "def f%d(): pass\n", i);
3131+
}
3132+
CBMFileResult *r = extract(src, CBM_LANG_PYTHON, "t", "big.py");
3133+
ASSERT_NOT_NULL(r);
3134+
ASSERT_FALSE(r->has_error);
3135+
int nfuncs = count_defs_with_label(r, "Function");
3136+
ASSERT(nfuncs >= N); /* all N present — not truncated at the old 4096 cap */
3137+
cbm_free_result(r);
3138+
free(src);
3139+
PASS();
3140+
}
3141+
31153142
/* ═══════════════════════════════════════════════════════════════════
31163143
* Suite
31173144
* ═══════════════════════════════════════════════════════════════════ */
@@ -3354,6 +3381,7 @@ SUITE(extraction) {
33543381
RUN_TEST(complexity_recursion_in_loop_unguarded);
33553382
RUN_TEST(complexity_guarded_recursion);
33563383
RUN_TEST(complexity_access_depth_and_params);
3384+
RUN_TEST(walk_defs_no_truncation_over_4096_issue668);
33573385

33583386
cbm_shutdown();
33593387
}

0 commit comments

Comments
 (0)