|
3 | 3 | #include "helpers.h" |
4 | 4 | #include "lang_specs.h" |
5 | 5 | #include "foundation/constants.h" |
6 | | -#include "foundation/platform.h" // safe_realloc (frees old on failure) |
7 | | -#include "foundation/log.h" // cbm_log_warn |
| 6 | +#include "foundation/log.h" // cbm_log_error |
| 7 | +#include "foundation/mem_core.h" // cbm_realloc/cbm_free -- walk_defs stack |
8 | 8 | #include "extract_node_stack.h" |
9 | 9 | #include "simhash/minhash.h" |
10 | 10 | #include "semantic/ast_profile.h" |
11 | 11 | #include "tree_sitter/api.h" // TSNode, ts_node_* |
12 | | -#include <stdint.h> // uint32_t |
| 12 | +#include <limits.h> // INT_MAX |
| 13 | +#include <stdint.h> // uint32_t, SIZE_MAX |
13 | 14 | #include <stdio.h> // snprintf (ObjectScript storage/trigger sidecars) |
14 | | -#include <stdlib.h> // getenv, atoi |
| 15 | +#include <stdlib.h> // malloc/free (child-collection scratch) |
15 | 16 | #include <string.h> |
16 | 17 | #include <ctype.h> |
17 | 18 |
|
@@ -7344,74 +7345,75 @@ typedef struct { |
7344 | 7345 | * single ~160 KB C-stack frame. That overflowed small thread stacks (the |
7345 | 7346 | * pre-2026-03 Windows 1 MB main thread) on the definitions pass, and its |
7346 | 7347 | * `top < 4096` push guards SILENTLY DROPPED every top-level definition past |
7347 | | - * 4096. Use a growable heap stack instead: a tiny initial footprint that doubles |
7348 | | - * on demand, bounded by a generous, env-configurable ceiling that WARNs (once) |
7349 | | - * rather than dropping — so a file with thousands of top-level defs is fully |
7350 | | - * extracted, and a pathological one degrades to a warned, bounded skip instead |
7351 | | - * of an OOM or a stack overflow. */ |
| 7348 | + * 4096. It became a growable stack bounded by an 8M-frame ceiling |
| 7349 | + * (env CBM_WALK_DEFS_MAX) that still stopped pushing once reached — and since |
| 7350 | + * children are pushed last-to-first, a file wider than the ceiling lost its |
| 7351 | + * FIRST definitions. No work cap may decide graph content, so there is no |
| 7352 | + * ceiling: the stack doubles on demand, and only an allocation failure stops |
| 7353 | + * the walk. That failure is loud — logged at error level and reported as a |
| 7354 | + * per-file extract error (result->has_error) — never a silent partial file. */ |
7352 | 7355 | typedef struct { |
7353 | 7356 | walk_defs_frame_t *data; |
7354 | 7357 | int top; |
7355 | 7358 | int cap; |
7356 | | - const char *path; // for the WARN when the ceiling is hit (may be NULL) |
7357 | | - bool warned; |
| 7359 | + const char *path; // for the error log on allocation failure (may be NULL) |
| 7360 | + bool failed; // growth failed: pending frames dropped, walk drains |
7358 | 7361 | /* The per-file traversal scratch (ctx->scratch) when there is one: frames |
7359 | 7362 | * then come from memory the thread reuses file after file, where a malloc |
7360 | 7363 | * of 256 frames per file was 28 k allocations and 255 MB never written on |
7361 | 7364 | * the Go corpus (waste sanitizer, 2026-09-17). Growth copies into a |
7362 | 7365 | * doubled buffer and abandons the old one to the arena, like TSNodeStack. |
7363 | | - * NULL: the heap, freed by the walk. */ |
| 7366 | + * NULL: the memory core (class EXTRACT), released by the walk. */ |
7364 | 7367 | CBMArena *arena; |
7365 | 7368 | } wd_stack_t; |
7366 | 7369 |
|
7367 | | -// Generous safety ceiling (frames), env-overridable via CBM_WALK_DEFS_MAX. |
7368 | | -// Realistic files never approach this; it only bounds a pathological/adversarial |
7369 | | -// file so extraction degrades to a warned skip rather than unbounded memory. |
7370 | | -static int wd_stack_max(void) { |
7371 | | - const char *e = getenv("CBM_WALK_DEFS_MAX"); |
7372 | | - if (e) { |
7373 | | - int v = atoi(e); |
7374 | | - if (v > 0) { |
7375 | | - return v; |
7376 | | - } |
7377 | | - } |
7378 | | - return 8 * 1024 * 1024; // 8M frames (~320 MB) default |
7379 | | -} |
7380 | | - |
7381 | | -static void wd_push(wd_stack_t *s, TSNode node, const char *enclosing_qn) { |
7382 | | - if (s->top >= s->cap) { |
7383 | | - int ncap = s->cap ? s->cap * 2 : 256; |
7384 | | - if (ncap > wd_stack_max()) { |
7385 | | - if (!s->warned) { |
7386 | | - char lim[24]; |
7387 | | - snprintf(lim, sizeof(lim), "%d", wd_stack_max()); |
7388 | | - cbm_log_warn("extract.walk_defs_capped", "limit", lim, "path", |
7389 | | - s->path ? s->path : ""); |
7390 | | - s->warned = true; |
7391 | | - } |
7392 | | - return; // bounded: stop growing (warned, not silent) |
7393 | | - } |
7394 | | - walk_defs_frame_t *nd = NULL; |
| 7370 | +enum { WD_STACK_INITIAL = 256 }; |
| 7371 | + |
| 7372 | +/* Double `s` (or give it its first WD_STACK_INITIAL frames). Returns false, |
| 7373 | + * after logging, when the doubled size is not representable or cannot be |
| 7374 | + * allocated; the old buffer is left intact so the walk can still release it. */ |
| 7375 | +static bool wd_grow(wd_stack_t *s) { |
| 7376 | + walk_defs_frame_t *nd = NULL; |
| 7377 | + int ncap = WD_STACK_INITIAL; |
| 7378 | + bool fits = true; |
| 7379 | + if (s->cap > 0) { |
| 7380 | + fits = s->cap <= INT_MAX / 2; |
| 7381 | + ncap = fits ? s->cap * 2 : s->cap; |
| 7382 | + } |
| 7383 | + fits = fits && (size_t)ncap <= SIZE_MAX / sizeof(walk_defs_frame_t); |
| 7384 | + if (fits) { |
| 7385 | + size_t bytes = (size_t)ncap * sizeof(walk_defs_frame_t); |
7395 | 7386 | if (s->arena) { |
7396 | | - nd = (walk_defs_frame_t *)cbm_arena_alloc(s->arena, |
7397 | | - (size_t)ncap * sizeof(walk_defs_frame_t)); |
| 7387 | + nd = (walk_defs_frame_t *)cbm_arena_alloc(s->arena, bytes); |
7398 | 7388 | if (nd && s->top > 0) { |
7399 | 7389 | memcpy(nd, s->data, (size_t)s->top * sizeof(walk_defs_frame_t)); |
7400 | 7390 | } |
7401 | 7391 | } else { |
7402 | | - nd = safe_realloc(s->data, (size_t)ncap * sizeof(walk_defs_frame_t)); |
7403 | | - } |
7404 | | - if (!nd) { |
7405 | | - /* OOM — safe_realloc already freed the old buffer. Bail cleanly: drop |
7406 | | - * pending frames so the walk_defs loop drains and exits without a NULL |
7407 | | - * deref; extraction keeps whatever was already emitted. */ |
7408 | | - s->data = NULL; |
7409 | | - s->cap = 0; |
7410 | | - s->top = 0; |
7411 | | - return; |
| 7392 | + nd = (walk_defs_frame_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, s->data, bytes); |
7412 | 7393 | } |
7413 | | - s->data = nd; |
7414 | | - s->cap = ncap; |
| 7394 | + } |
| 7395 | + if (!nd) { |
| 7396 | + char pending[24]; |
| 7397 | + snprintf(pending, sizeof(pending), "%d", s->top); |
| 7398 | + cbm_log_error("extract.walk_stack_alloc_failed", "walker", "walk_defs", "pending", pending, |
| 7399 | + "path", s->path ? s->path : ""); |
| 7400 | + return false; |
| 7401 | + } |
| 7402 | + s->data = nd; |
| 7403 | + s->cap = ncap; |
| 7404 | + return true; |
| 7405 | +} |
| 7406 | + |
| 7407 | +static void wd_push(wd_stack_t *s, TSNode node, const char *enclosing_qn) { |
| 7408 | + if (s->failed) { |
| 7409 | + return; |
| 7410 | + } |
| 7411 | + if (s->top >= s->cap && !wd_grow(s)) { |
| 7412 | + /* Drop every pending frame so the walk_defs loop drains and exits; |
| 7413 | + * walk_defs then marks the file's result as an extract error. */ |
| 7414 | + s->failed = true; |
| 7415 | + s->top = 0; |
| 7416 | + return; |
7415 | 7417 | } |
7416 | 7418 | s->data[s->top++] = (walk_defs_frame_t){node, enclosing_qn}; |
7417 | 7419 | } |
@@ -8219,7 +8221,11 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec, |
8219 | 8221 | wd_push_children_reverse(&s, node, frame.enclosing_class_qn); |
8220 | 8222 | } |
8221 | 8223 | if (!s.arena) { |
8222 | | - free(s.data); |
| 8224 | + cbm_free(CBM_MEM_CLASS_EXTRACT, s.data); |
| 8225 | + } |
| 8226 | + if (s.failed) { |
| 8227 | + ctx->result->has_error = true; |
| 8228 | + ctx->result->error_msg = "definitions walk: stack allocation failed"; |
8223 | 8229 | } |
8224 | 8230 | } |
8225 | 8231 |
|
|
0 commit comments