Skip to content

Commit f4654a7

Browse files
authored
Merge pull request DeusData#2373 from DeusData/fix/defs-walk-no-cap
fix(extract): definitions walk stack grows without a ceiling
2 parents b0fe8ae + 77ab4be commit f4654a7

3 files changed

Lines changed: 95 additions & 56 deletions

File tree

‎internal/cbm/extract_defs.c‎

Lines changed: 61 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,16 @@
33
#include "helpers.h"
44
#include "lang_specs.h"
55
#include "foundation/constants.h"
6-
#include "foundation/platform.h" // safe_realloc (frees old on failure)
7-
#include "foundation/log.h" // cbm_log_warn
6+
#include "foundation/log.h" // cbm_log_error
7+
#include "foundation/mem_core.h" // cbm_realloc/cbm_free -- walk_defs stack
88
#include "extract_node_stack.h"
99
#include "simhash/minhash.h"
1010
#include "semantic/ast_profile.h"
1111
#include "tree_sitter/api.h" // TSNode, ts_node_*
12-
#include <stdint.h> // uint32_t
12+
#include <limits.h> // INT_MAX
13+
#include <stdint.h> // uint32_t, SIZE_MAX
1314
#include <stdio.h> // snprintf (ObjectScript storage/trigger sidecars)
14-
#include <stdlib.h> // getenv, atoi
15+
#include <stdlib.h> // malloc/free (child-collection scratch)
1516
#include <string.h>
1617
#include <ctype.h>
1718

@@ -7209,74 +7210,75 @@ typedef struct {
72097210
* single ~160 KB C-stack frame. That overflowed small thread stacks (the
72107211
* pre-2026-03 Windows 1 MB main thread) on the definitions pass, and its
72117212
* `top < 4096` push guards SILENTLY DROPPED every top-level definition past
7212-
* 4096. Use a growable heap stack instead: a tiny initial footprint that doubles
7213-
* on demand, bounded by a generous, env-configurable ceiling that WARNs (once)
7214-
* rather than dropping — so a file with thousands of top-level defs is fully
7215-
* extracted, and a pathological one degrades to a warned, bounded skip instead
7216-
* of an OOM or a stack overflow. */
7213+
* 4096. It became a growable stack bounded by an 8M-frame ceiling
7214+
* (env CBM_WALK_DEFS_MAX) that still stopped pushing once reached — and since
7215+
* children are pushed last-to-first, a file wider than the ceiling lost its
7216+
* FIRST definitions. No work cap may decide graph content, so there is no
7217+
* ceiling: the stack doubles on demand, and only an allocation failure stops
7218+
* the walk. That failure is loud — logged at error level and reported as a
7219+
* per-file extract error (result->has_error) — never a silent partial file. */
72177220
typedef struct {
72187221
walk_defs_frame_t *data;
72197222
int top;
72207223
int cap;
7221-
const char *path; // for the WARN when the ceiling is hit (may be NULL)
7222-
bool warned;
7224+
const char *path; // for the error log on allocation failure (may be NULL)
7225+
bool failed; // growth failed: pending frames dropped, walk drains
72237226
/* The per-file traversal scratch (ctx->scratch) when there is one: frames
72247227
* then come from memory the thread reuses file after file, where a malloc
72257228
* of 256 frames per file was 28 k allocations and 255 MB never written on
72267229
* the Go corpus (waste sanitizer, 2026-09-17). Growth copies into a
72277230
* doubled buffer and abandons the old one to the arena, like TSNodeStack.
7228-
* NULL: the heap, freed by the walk. */
7231+
* NULL: the memory core (class EXTRACT), released by the walk. */
72297232
CBMArena *arena;
72307233
} wd_stack_t;
72317234

7232-
// Generous safety ceiling (frames), env-overridable via CBM_WALK_DEFS_MAX.
7233-
// Realistic files never approach this; it only bounds a pathological/adversarial
7234-
// file so extraction degrades to a warned skip rather than unbounded memory.
7235-
static int wd_stack_max(void) {
7236-
const char *e = getenv("CBM_WALK_DEFS_MAX");
7237-
if (e) {
7238-
int v = atoi(e);
7239-
if (v > 0) {
7240-
return v;
7241-
}
7242-
}
7243-
return 8 * 1024 * 1024; // 8M frames (~320 MB) default
7244-
}
7245-
7246-
static void wd_push(wd_stack_t *s, TSNode node, const char *enclosing_qn) {
7247-
if (s->top >= s->cap) {
7248-
int ncap = s->cap ? s->cap * 2 : 256;
7249-
if (ncap > wd_stack_max()) {
7250-
if (!s->warned) {
7251-
char lim[24];
7252-
snprintf(lim, sizeof(lim), "%d", wd_stack_max());
7253-
cbm_log_warn("extract.walk_defs_capped", "limit", lim, "path",
7254-
s->path ? s->path : "");
7255-
s->warned = true;
7256-
}
7257-
return; // bounded: stop growing (warned, not silent)
7258-
}
7259-
walk_defs_frame_t *nd = NULL;
7235+
enum { WD_STACK_INITIAL = 256 };
7236+
7237+
/* Double `s` (or give it its first WD_STACK_INITIAL frames). Returns false,
7238+
* after logging, when the doubled size is not representable or cannot be
7239+
* allocated; the old buffer is left intact so the walk can still release it. */
7240+
static bool wd_grow(wd_stack_t *s) {
7241+
walk_defs_frame_t *nd = NULL;
7242+
int ncap = WD_STACK_INITIAL;
7243+
bool fits = true;
7244+
if (s->cap > 0) {
7245+
fits = s->cap <= INT_MAX / 2;
7246+
ncap = fits ? s->cap * 2 : s->cap;
7247+
}
7248+
fits = fits && (size_t)ncap <= SIZE_MAX / sizeof(walk_defs_frame_t);
7249+
if (fits) {
7250+
size_t bytes = (size_t)ncap * sizeof(walk_defs_frame_t);
72607251
if (s->arena) {
7261-
nd = (walk_defs_frame_t *)cbm_arena_alloc(s->arena,
7262-
(size_t)ncap * sizeof(walk_defs_frame_t));
7252+
nd = (walk_defs_frame_t *)cbm_arena_alloc(s->arena, bytes);
72637253
if (nd && s->top > 0) {
72647254
memcpy(nd, s->data, (size_t)s->top * sizeof(walk_defs_frame_t));
72657255
}
72667256
} else {
7267-
nd = safe_realloc(s->data, (size_t)ncap * sizeof(walk_defs_frame_t));
7268-
}
7269-
if (!nd) {
7270-
/* OOM — safe_realloc already freed the old buffer. Bail cleanly: drop
7271-
* pending frames so the walk_defs loop drains and exits without a NULL
7272-
* deref; extraction keeps whatever was already emitted. */
7273-
s->data = NULL;
7274-
s->cap = 0;
7275-
s->top = 0;
7276-
return;
7257+
nd = (walk_defs_frame_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, s->data, bytes);
72777258
}
7278-
s->data = nd;
7279-
s->cap = ncap;
7259+
}
7260+
if (!nd) {
7261+
char pending[24];
7262+
snprintf(pending, sizeof(pending), "%d", s->top);
7263+
cbm_log_error("extract.walk_stack_alloc_failed", "walker", "walk_defs", "pending", pending,
7264+
"path", s->path ? s->path : "");
7265+
return false;
7266+
}
7267+
s->data = nd;
7268+
s->cap = ncap;
7269+
return true;
7270+
}
7271+
7272+
static void wd_push(wd_stack_t *s, TSNode node, const char *enclosing_qn) {
7273+
if (s->failed) {
7274+
return;
7275+
}
7276+
if (s->top >= s->cap && !wd_grow(s)) {
7277+
/* Drop every pending frame so the walk_defs loop drains and exits;
7278+
* walk_defs then marks the file's result as an extract error. */
7279+
s->failed = true;
7280+
s->top = 0;
7281+
return;
72807282
}
72817283
s->data[s->top++] = (walk_defs_frame_t){node, enclosing_qn};
72827284
}
@@ -8084,7 +8086,11 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
80848086
wd_push_children_reverse(&s, node, frame.enclosing_class_qn);
80858087
}
80868088
if (!s.arena) {
8087-
free(s.data);
8089+
cbm_free(CBM_MEM_CLASS_EXTRACT, s.data);
8090+
}
8091+
if (s.failed) {
8092+
ctx->result->has_error = true;
8093+
ctx->result->error_msg = "definitions walk: stack allocation failed";
80888094
}
80898095
}
80908096

‎scripts/memory-core-baseline.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
internal/cbm/ac.c 20
77
internal/cbm/arena.c 3
88
internal/cbm/cbm.c 13
9-
internal/cbm/extract_defs.c 5
9+
internal/cbm/extract_defs.c 4
1010
internal/cbm/helpers.c 3
1111
internal/cbm/lsp/c_lsp.c 4
1212
internal/cbm/lsp/perl_lsp.c 21

‎tests/test_extraction.c‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5480,6 +5480,38 @@ TEST(complexity_access_depth_and_params) {
54805480
PASS();
54815481
}
54825482

5483+
/* The definitions walk kept its pending frames under a ceiling (8M frames,
5484+
* env CBM_WALK_DEFS_MAX) and stopped pushing once it was reached. Children are
5485+
* pushed last-to-first, so a file wider than the ceiling lost its FIRST
5486+
* top-level definitions. The ceiling and its env knob are gone; the knob is set
5487+
* here to a value this file exceeds to prove it no longer decides content. */
5488+
enum { WIDE_DEFS = 1000, WIDE_DEFS_OLD_CAP = 256 };
5489+
5490+
TEST(walk_defs_wide_file_extracts_every_definition) {
5491+
size_t cap = (size_t)WIDE_DEFS * 32 + 64;
5492+
char *src = malloc(cap);
5493+
ASSERT_NOT_NULL(src);
5494+
size_t pos = 0;
5495+
for (int i = 0; i < WIDE_DEFS; i++) {
5496+
pos += (size_t)snprintf(src + pos, cap - pos, "int wd%d(void) { return %d; }\n", i, i);
5497+
}
5498+
char lim[16];
5499+
snprintf(lim, sizeof(lim), "%d", WIDE_DEFS_OLD_CAP);
5500+
cbm_setenv("CBM_WALK_DEFS_MAX", lim, 1);
5501+
CBMFileResult *r = extract(src, CBM_LANG_C, "t", "wide_defs.c");
5502+
cbm_unsetenv("CBM_WALK_DEFS_MAX");
5503+
ASSERT_NOT_NULL(r);
5504+
ASSERT_FALSE(r->has_error);
5505+
for (int i = 0; i < WIDE_DEFS; i++) {
5506+
char name[16];
5507+
snprintf(name, sizeof(name), "wd%d", i);
5508+
ASSERT_NOT_NULL(find_def(r, name));
5509+
}
5510+
cbm_free_result(r);
5511+
free(src);
5512+
PASS();
5513+
}
5514+
54835515
/* ═══════════════════════════════════════════════════════════════════
54845516
* Perl call-graph noise (#459 follow-up)
54855517
* ═══════════════════════════════════════════════════════════════════ */
@@ -8738,6 +8770,7 @@ SUITE(extraction) {
87388770
RUN_TEST(complexity_go_method_receiver_self_recursion);
87398771
RUN_TEST(complexity_delegation_receivers_not_recursive_issue876);
87408772
RUN_TEST(complexity_access_depth_and_params);
8773+
RUN_TEST(walk_defs_wide_file_extracts_every_definition);
87418774
RUN_TEST(extract_c_ifdef_split_brace_fn_recovered_issue961);
87428775
RUN_TEST(extract_cpp_preproc_signature_gap_issue946);
87438776
RUN_TEST(extract_cpp_preproc_macro_generated_callable_skipped_issue949);

0 commit comments

Comments
 (0)