|
3 | 3 | #include "helpers.h" |
4 | 4 | #include "lang_specs.h" |
5 | 5 | #include "foundation/constants.h" |
6 | | -#include "foundation/platform.h" // safe_realloc (frees old on failure) |
7 | | -#include "foundation/log.h" // cbm_log_warn |
| 6 | +#include "foundation/log.h" // cbm_log_error |
| 7 | +#include "foundation/mem_core.h" // cbm_realloc/cbm_free -- walk_defs stack |
8 | 8 | #include "extract_node_stack.h" |
9 | 9 | #include "simhash/minhash.h" |
10 | 10 | #include "semantic/ast_profile.h" |
11 | 11 | #include "tree_sitter/api.h" // TSNode, ts_node_* |
12 | | -#include <stdint.h> // uint32_t |
| 12 | +#include <limits.h> // INT_MAX |
| 13 | +#include <stdint.h> // uint32_t, SIZE_MAX |
13 | 14 | #include <stdio.h> // snprintf (ObjectScript storage/trigger sidecars) |
14 | | -#include <stdlib.h> // getenv, atoi |
| 15 | +#include <stdlib.h> // malloc/free (child-collection scratch) |
15 | 16 | #include <string.h> |
16 | 17 | #include <ctype.h> |
17 | 18 |
|
@@ -7209,74 +7210,75 @@ typedef struct { |
7209 | 7210 | * single ~160 KB C-stack frame. That overflowed small thread stacks (the |
7210 | 7211 | * pre-2026-03 Windows 1 MB main thread) on the definitions pass, and its |
7211 | 7212 | * `top < 4096` push guards SILENTLY DROPPED every top-level definition past |
7212 | | - * 4096. Use a growable heap stack instead: a tiny initial footprint that doubles |
7213 | | - * on demand, bounded by a generous, env-configurable ceiling that WARNs (once) |
7214 | | - * rather than dropping — so a file with thousands of top-level defs is fully |
7215 | | - * extracted, and a pathological one degrades to a warned, bounded skip instead |
7216 | | - * of an OOM or a stack overflow. */ |
| 7213 | + * 4096. It became a growable stack bounded by an 8M-frame ceiling |
| 7214 | + * (env CBM_WALK_DEFS_MAX) that still stopped pushing once reached — and since |
| 7215 | + * children are pushed last-to-first, a file wider than the ceiling lost its |
| 7216 | + * FIRST definitions. No work cap may decide graph content, so there is no |
| 7217 | + * ceiling: the stack doubles on demand, and only an allocation failure stops |
| 7218 | + * the walk. That failure is loud — logged at error level and reported as a |
| 7219 | + * per-file extract error (result->has_error) — never a silent partial file. */ |
7217 | 7220 | typedef struct { |
7218 | 7221 | walk_defs_frame_t *data; |
7219 | 7222 | int top; |
7220 | 7223 | int cap; |
7221 | | - const char *path; // for the WARN when the ceiling is hit (may be NULL) |
7222 | | - bool warned; |
| 7224 | + const char *path; // for the error log on allocation failure (may be NULL) |
| 7225 | + bool failed; // growth failed: pending frames dropped, walk drains |
7223 | 7226 | /* The per-file traversal scratch (ctx->scratch) when there is one: frames |
7224 | 7227 | * then come from memory the thread reuses file after file, where a malloc |
7225 | 7228 | * of 256 frames per file was 28 k allocations and 255 MB never written on |
7226 | 7229 | * the Go corpus (waste sanitizer, 2026-09-17). Growth copies into a |
7227 | 7230 | * doubled buffer and abandons the old one to the arena, like TSNodeStack. |
7228 | | - * NULL: the heap, freed by the walk. */ |
| 7231 | + * NULL: the memory core (class EXTRACT), released by the walk. */ |
7229 | 7232 | CBMArena *arena; |
7230 | 7233 | } wd_stack_t; |
7231 | 7234 |
|
7232 | | -// Generous safety ceiling (frames), env-overridable via CBM_WALK_DEFS_MAX. |
7233 | | -// Realistic files never approach this; it only bounds a pathological/adversarial |
7234 | | -// file so extraction degrades to a warned skip rather than unbounded memory. |
7235 | | -static int wd_stack_max(void) { |
7236 | | - const char *e = getenv("CBM_WALK_DEFS_MAX"); |
7237 | | - if (e) { |
7238 | | - int v = atoi(e); |
7239 | | - if (v > 0) { |
7240 | | - return v; |
7241 | | - } |
7242 | | - } |
7243 | | - return 8 * 1024 * 1024; // 8M frames (~320 MB) default |
7244 | | -} |
7245 | | - |
7246 | | -static void wd_push(wd_stack_t *s, TSNode node, const char *enclosing_qn) { |
7247 | | - if (s->top >= s->cap) { |
7248 | | - int ncap = s->cap ? s->cap * 2 : 256; |
7249 | | - if (ncap > wd_stack_max()) { |
7250 | | - if (!s->warned) { |
7251 | | - char lim[24]; |
7252 | | - snprintf(lim, sizeof(lim), "%d", wd_stack_max()); |
7253 | | - cbm_log_warn("extract.walk_defs_capped", "limit", lim, "path", |
7254 | | - s->path ? s->path : ""); |
7255 | | - s->warned = true; |
7256 | | - } |
7257 | | - return; // bounded: stop growing (warned, not silent) |
7258 | | - } |
7259 | | - walk_defs_frame_t *nd = NULL; |
| 7235 | +enum { WD_STACK_INITIAL = 256 }; |
| 7236 | + |
| 7237 | +/* Double `s` (or give it its first WD_STACK_INITIAL frames). Returns false, |
| 7238 | + * after logging, when the doubled size is not representable or cannot be |
| 7239 | + * allocated; the old buffer is left intact so the walk can still release it. */ |
| 7240 | +static bool wd_grow(wd_stack_t *s) { |
| 7241 | + walk_defs_frame_t *nd = NULL; |
| 7242 | + int ncap = WD_STACK_INITIAL; |
| 7243 | + bool fits = true; |
| 7244 | + if (s->cap > 0) { |
| 7245 | + fits = s->cap <= INT_MAX / 2; |
| 7246 | + ncap = fits ? s->cap * 2 : s->cap; |
| 7247 | + } |
| 7248 | + fits = fits && (size_t)ncap <= SIZE_MAX / sizeof(walk_defs_frame_t); |
| 7249 | + if (fits) { |
| 7250 | + size_t bytes = (size_t)ncap * sizeof(walk_defs_frame_t); |
7260 | 7251 | if (s->arena) { |
7261 | | - nd = (walk_defs_frame_t *)cbm_arena_alloc(s->arena, |
7262 | | - (size_t)ncap * sizeof(walk_defs_frame_t)); |
| 7252 | + nd = (walk_defs_frame_t *)cbm_arena_alloc(s->arena, bytes); |
7263 | 7253 | if (nd && s->top > 0) { |
7264 | 7254 | memcpy(nd, s->data, (size_t)s->top * sizeof(walk_defs_frame_t)); |
7265 | 7255 | } |
7266 | 7256 | } else { |
7267 | | - nd = safe_realloc(s->data, (size_t)ncap * sizeof(walk_defs_frame_t)); |
7268 | | - } |
7269 | | - if (!nd) { |
7270 | | - /* OOM — safe_realloc already freed the old buffer. Bail cleanly: drop |
7271 | | - * pending frames so the walk_defs loop drains and exits without a NULL |
7272 | | - * deref; extraction keeps whatever was already emitted. */ |
7273 | | - s->data = NULL; |
7274 | | - s->cap = 0; |
7275 | | - s->top = 0; |
7276 | | - return; |
| 7257 | + nd = (walk_defs_frame_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, s->data, bytes); |
7277 | 7258 | } |
7278 | | - s->data = nd; |
7279 | | - s->cap = ncap; |
| 7259 | + } |
| 7260 | + if (!nd) { |
| 7261 | + char pending[24]; |
| 7262 | + snprintf(pending, sizeof(pending), "%d", s->top); |
| 7263 | + cbm_log_error("extract.walk_stack_alloc_failed", "walker", "walk_defs", "pending", pending, |
| 7264 | + "path", s->path ? s->path : ""); |
| 7265 | + return false; |
| 7266 | + } |
| 7267 | + s->data = nd; |
| 7268 | + s->cap = ncap; |
| 7269 | + return true; |
| 7270 | +} |
| 7271 | + |
| 7272 | +static void wd_push(wd_stack_t *s, TSNode node, const char *enclosing_qn) { |
| 7273 | + if (s->failed) { |
| 7274 | + return; |
| 7275 | + } |
| 7276 | + if (s->top >= s->cap && !wd_grow(s)) { |
| 7277 | + /* Drop every pending frame so the walk_defs loop drains and exits; |
| 7278 | + * walk_defs then marks the file's result as an extract error. */ |
| 7279 | + s->failed = true; |
| 7280 | + s->top = 0; |
| 7281 | + return; |
7280 | 7282 | } |
7281 | 7283 | s->data[s->top++] = (walk_defs_frame_t){node, enclosing_qn}; |
7282 | 7284 | } |
@@ -8084,7 +8086,11 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec, |
8084 | 8086 | wd_push_children_reverse(&s, node, frame.enclosing_class_qn); |
8085 | 8087 | } |
8086 | 8088 | if (!s.arena) { |
8087 | | - free(s.data); |
| 8089 | + cbm_free(CBM_MEM_CLASS_EXTRACT, s.data); |
| 8090 | + } |
| 8091 | + if (s.failed) { |
| 8092 | + ctx->result->has_error = true; |
| 8093 | + ctx->result->error_msg = "definitions walk: stack allocation failed"; |
8088 | 8094 | } |
8089 | 8095 | } |
8090 | 8096 |
|
|
0 commit comments