Skip to content

Commit 77ab4be

Browse files
committed
fix(extract): definitions walk stack grows without a ceiling
walk_defs keeps its pending frames in a growable stack that was bounded by an 8M-frame ceiling (env CBM_WALK_DEFS_MAX). At the ceiling wd_push logged "extract.walk_defs_capped" once and then stopped pushing. Children are pushed last-to-first, so a file wider than the ceiling lost its FIRST definitions: a work cap decided graph content. Fix: no ceiling. The stack doubles on demand, heap frames now come from the memory core (cbm_realloc/cbm_free, class EXTRACT) instead of safe_realloc and raw free, and the doubling is overflow-guarded (int capacity and size_t byte count). The only thing that can stop the walk is an allocation failure, and that is loud: logged at error level as "extract.walk_stack_alloc_failed" (walker=walk_defs, pending, path) -- the same event the complexity walkers use -- and the file's result is marked has_error with "definitions walk: stack allocation failed", which the definitions passes already report as a per-file extract error. The arena-backed path (ctx->scratch) gets the same failure handling. Traversal order is unchanged. CBM_WALK_DEFS_MAX is removed. It was read only in wd_stack_max(); it was not documented anywhere (docs, --help, config, tests), so nothing else changes. extract_defs.c loses its raw free() call, so the memory-core baseline for the file drops from 5 to 4. Test (extraction suite): walk_defs_wide_file_extracts_every_definition extracts a C file with 1000 top-level functions with CBM_WALK_DEFS_MAX=256 set, and asserts every wd0..wd999 definition is present. RED on main (extract.walk_defs_capped limit=256; wd0 missing), GREEN with the fix, RED again with the production change reverted. The extraction (379), complexity (5), ast_profile (10) and semantic (35) suites pass. Real input: Linux kernel, index-only, fresh cache and runtime dirs, same machine. main and fix both produce 8,530,336 nodes and 15,764,128 edges, SEMANTICALLY_RELATED 607 each, and the full node set (label, QN, lines) and edge set (type, source QN, target QN) hash identically -- no kernel file approaches the old ceiling. Index wall time 210.0 s (main) vs 194.4 s (fix), within run-to-run noise. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 64c23fa commit 77ab4be

3 files changed

Lines changed: 95 additions & 56 deletions

File tree

‎internal/cbm/extract_defs.c‎

Lines changed: 61 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,16 @@
33
#include "helpers.h"
44
#include "lang_specs.h"
55
#include "foundation/constants.h"
6-
#include "foundation/platform.h" // safe_realloc (frees old on failure)
7-
#include "foundation/log.h" // cbm_log_warn
6+
#include "foundation/log.h" // cbm_log_error
7+
#include "foundation/mem_core.h" // cbm_realloc/cbm_free -- walk_defs stack
88
#include "extract_node_stack.h"
99
#include "simhash/minhash.h"
1010
#include "semantic/ast_profile.h"
1111
#include "tree_sitter/api.h" // TSNode, ts_node_*
12-
#include <stdint.h> // uint32_t
12+
#include <limits.h> // INT_MAX
13+
#include <stdint.h> // uint32_t, SIZE_MAX
1314
#include <stdio.h> // snprintf (ObjectScript storage/trigger sidecars)
14-
#include <stdlib.h> // getenv, atoi
15+
#include <stdlib.h> // malloc/free (child-collection scratch)
1516
#include <string.h>
1617
#include <ctype.h>
1718

@@ -7195,74 +7196,75 @@ typedef struct {
71957196
* single ~160 KB C-stack frame. That overflowed small thread stacks (the
71967197
* pre-2026-03 Windows 1 MB main thread) on the definitions pass, and its
71977198
* `top < 4096` push guards SILENTLY DROPPED every top-level definition past
7198-
* 4096. Use a growable heap stack instead: a tiny initial footprint that doubles
7199-
* on demand, bounded by a generous, env-configurable ceiling that WARNs (once)
7200-
* rather than dropping — so a file with thousands of top-level defs is fully
7201-
* extracted, and a pathological one degrades to a warned, bounded skip instead
7202-
* of an OOM or a stack overflow. */
7199+
* 4096. It became a growable stack bounded by an 8M-frame ceiling
7200+
* (env CBM_WALK_DEFS_MAX) that still stopped pushing once reached — and since
7201+
* children are pushed last-to-first, a file wider than the ceiling lost its
7202+
* FIRST definitions. No work cap may decide graph content, so there is no
7203+
* ceiling: the stack doubles on demand, and only an allocation failure stops
7204+
* the walk. That failure is loud — logged at error level and reported as a
7205+
* per-file extract error (result->has_error) — never a silent partial file. */
72037206
typedef struct {
72047207
walk_defs_frame_t *data;
72057208
int top;
72067209
int cap;
7207-
const char *path; // for the WARN when the ceiling is hit (may be NULL)
7208-
bool warned;
7210+
const char *path; // for the error log on allocation failure (may be NULL)
7211+
bool failed; // growth failed: pending frames dropped, walk drains
72097212
/* The per-file traversal scratch (ctx->scratch) when there is one: frames
72107213
* then come from memory the thread reuses file after file, where a malloc
72117214
* of 256 frames per file was 28 k allocations and 255 MB never written on
72127215
* the Go corpus (waste sanitizer, 2026-09-17). Growth copies into a
72137216
* doubled buffer and abandons the old one to the arena, like TSNodeStack.
7214-
* NULL: the heap, freed by the walk. */
7217+
* NULL: the memory core (class EXTRACT), released by the walk. */
72157218
CBMArena *arena;
72167219
} wd_stack_t;
72177220

7218-
// Generous safety ceiling (frames), env-overridable via CBM_WALK_DEFS_MAX.
7219-
// Realistic files never approach this; it only bounds a pathological/adversarial
7220-
// file so extraction degrades to a warned skip rather than unbounded memory.
7221-
static int wd_stack_max(void) {
7222-
const char *e = getenv("CBM_WALK_DEFS_MAX");
7223-
if (e) {
7224-
int v = atoi(e);
7225-
if (v > 0) {
7226-
return v;
7227-
}
7228-
}
7229-
return 8 * 1024 * 1024; // 8M frames (~320 MB) default
7230-
}
7231-
7232-
static void wd_push(wd_stack_t *s, TSNode node, const char *enclosing_qn) {
7233-
if (s->top >= s->cap) {
7234-
int ncap = s->cap ? s->cap * 2 : 256;
7235-
if (ncap > wd_stack_max()) {
7236-
if (!s->warned) {
7237-
char lim[24];
7238-
snprintf(lim, sizeof(lim), "%d", wd_stack_max());
7239-
cbm_log_warn("extract.walk_defs_capped", "limit", lim, "path",
7240-
s->path ? s->path : "");
7241-
s->warned = true;
7242-
}
7243-
return; // bounded: stop growing (warned, not silent)
7244-
}
7245-
walk_defs_frame_t *nd = NULL;
7221+
enum { WD_STACK_INITIAL = 256 };
7222+
7223+
/* Double `s` (or give it its first WD_STACK_INITIAL frames). Returns false,
7224+
* after logging, when the doubled size is not representable or cannot be
7225+
* allocated; the old buffer is left intact so the walk can still release it. */
7226+
static bool wd_grow(wd_stack_t *s) {
7227+
walk_defs_frame_t *nd = NULL;
7228+
int ncap = WD_STACK_INITIAL;
7229+
bool fits = true;
7230+
if (s->cap > 0) {
7231+
fits = s->cap <= INT_MAX / 2;
7232+
ncap = fits ? s->cap * 2 : s->cap;
7233+
}
7234+
fits = fits && (size_t)ncap <= SIZE_MAX / sizeof(walk_defs_frame_t);
7235+
if (fits) {
7236+
size_t bytes = (size_t)ncap * sizeof(walk_defs_frame_t);
72467237
if (s->arena) {
7247-
nd = (walk_defs_frame_t *)cbm_arena_alloc(s->arena,
7248-
(size_t)ncap * sizeof(walk_defs_frame_t));
7238+
nd = (walk_defs_frame_t *)cbm_arena_alloc(s->arena, bytes);
72497239
if (nd && s->top > 0) {
72507240
memcpy(nd, s->data, (size_t)s->top * sizeof(walk_defs_frame_t));
72517241
}
72527242
} else {
7253-
nd = safe_realloc(s->data, (size_t)ncap * sizeof(walk_defs_frame_t));
7254-
}
7255-
if (!nd) {
7256-
/* OOM — safe_realloc already freed the old buffer. Bail cleanly: drop
7257-
* pending frames so the walk_defs loop drains and exits without a NULL
7258-
* deref; extraction keeps whatever was already emitted. */
7259-
s->data = NULL;
7260-
s->cap = 0;
7261-
s->top = 0;
7262-
return;
7243+
nd = (walk_defs_frame_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, s->data, bytes);
72637244
}
7264-
s->data = nd;
7265-
s->cap = ncap;
7245+
}
7246+
if (!nd) {
7247+
char pending[24];
7248+
snprintf(pending, sizeof(pending), "%d", s->top);
7249+
cbm_log_error("extract.walk_stack_alloc_failed", "walker", "walk_defs", "pending", pending,
7250+
"path", s->path ? s->path : "");
7251+
return false;
7252+
}
7253+
s->data = nd;
7254+
s->cap = ncap;
7255+
return true;
7256+
}
7257+
7258+
static void wd_push(wd_stack_t *s, TSNode node, const char *enclosing_qn) {
7259+
if (s->failed) {
7260+
return;
7261+
}
7262+
if (s->top >= s->cap && !wd_grow(s)) {
7263+
/* Drop every pending frame so the walk_defs loop drains and exits;
7264+
* walk_defs then marks the file's result as an extract error. */
7265+
s->failed = true;
7266+
s->top = 0;
7267+
return;
72667268
}
72677269
s->data[s->top++] = (walk_defs_frame_t){node, enclosing_qn};
72687270
}
@@ -8070,7 +8072,11 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
80708072
wd_push_children_reverse(&s, node, frame.enclosing_class_qn);
80718073
}
80728074
if (!s.arena) {
8073-
free(s.data);
8075+
cbm_free(CBM_MEM_CLASS_EXTRACT, s.data);
8076+
}
8077+
if (s.failed) {
8078+
ctx->result->has_error = true;
8079+
ctx->result->error_msg = "definitions walk: stack allocation failed";
80748080
}
80758081
}
80768082

‎scripts/memory-core-baseline.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
internal/cbm/ac.c 20
77
internal/cbm/arena.c 3
88
internal/cbm/cbm.c 13
9-
internal/cbm/extract_defs.c 5
9+
internal/cbm/extract_defs.c 4
1010
internal/cbm/helpers.c 3
1111
internal/cbm/lsp/c_lsp.c 4
1212
internal/cbm/lsp/perl_lsp.c 21

‎tests/test_extraction.c‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5380,6 +5380,38 @@ TEST(complexity_access_depth_and_params) {
53805380
PASS();
53815381
}
53825382

5383+
/* The definitions walk kept its pending frames under a ceiling (8M frames,
5384+
* env CBM_WALK_DEFS_MAX) and stopped pushing once it was reached. Children are
5385+
* pushed last-to-first, so a file wider than the ceiling lost its FIRST
5386+
* top-level definitions. The ceiling and its env knob are gone; the knob is set
5387+
* here to a value this file exceeds to prove it no longer decides content. */
5388+
enum { WIDE_DEFS = 1000, WIDE_DEFS_OLD_CAP = 256 };
5389+
5390+
TEST(walk_defs_wide_file_extracts_every_definition) {
5391+
size_t cap = (size_t)WIDE_DEFS * 32 + 64;
5392+
char *src = malloc(cap);
5393+
ASSERT_NOT_NULL(src);
5394+
size_t pos = 0;
5395+
for (int i = 0; i < WIDE_DEFS; i++) {
5396+
pos += (size_t)snprintf(src + pos, cap - pos, "int wd%d(void) { return %d; }\n", i, i);
5397+
}
5398+
char lim[16];
5399+
snprintf(lim, sizeof(lim), "%d", WIDE_DEFS_OLD_CAP);
5400+
cbm_setenv("CBM_WALK_DEFS_MAX", lim, 1);
5401+
CBMFileResult *r = extract(src, CBM_LANG_C, "t", "wide_defs.c");
5402+
cbm_unsetenv("CBM_WALK_DEFS_MAX");
5403+
ASSERT_NOT_NULL(r);
5404+
ASSERT_FALSE(r->has_error);
5405+
for (int i = 0; i < WIDE_DEFS; i++) {
5406+
char name[16];
5407+
snprintf(name, sizeof(name), "wd%d", i);
5408+
ASSERT_NOT_NULL(find_def(r, name));
5409+
}
5410+
cbm_free_result(r);
5411+
free(src);
5412+
PASS();
5413+
}
5414+
53835415
/* ═══════════════════════════════════════════════════════════════════
53845416
* Perl call-graph noise (#459 follow-up)
53855417
* ═══════════════════════════════════════════════════════════════════ */
@@ -8635,6 +8667,7 @@ SUITE(extraction) {
86358667
RUN_TEST(complexity_go_method_receiver_self_recursion);
86368668
RUN_TEST(complexity_delegation_receivers_not_recursive_issue876);
86378669
RUN_TEST(complexity_access_depth_and_params);
8670+
RUN_TEST(walk_defs_wide_file_extracts_every_definition);
86388671
RUN_TEST(extract_c_ifdef_split_brace_fn_recovered_issue961);
86398672
RUN_TEST(extract_cpp_preproc_signature_gap_issue946);
86408673
RUN_TEST(extract_cpp_preproc_macro_generated_callable_skipped_issue949);

0 commit comments

Comments
 (0)