Skip to content

Commit 9871e7e

Browse files
authored
Merge pull request DeusData#827 from DeusData/feat/index-supervisor-subprocess
feat(index): crash/hang supervisor — quarantine the culprit file, index the rest
2 parents 836f90d + 6e59736 commit 9871e7e

17 files changed

Lines changed: 1535 additions & 11 deletions

‎Makefile.cbm‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,8 @@ FOUNDATION_SRCS = \
122122
src/foundation/diagnostics.c \
123123
src/foundation/profile.c \
124124
src/foundation/dump_verify.c \
125-
src/foundation/limits.c
125+
src/foundation/limits.c \
126+
src/foundation/subprocess.c
126127

127128
# Existing extraction C code (compiled from current location)
128129
EXTRACTION_SRCS = \
@@ -182,7 +183,7 @@ STORE_SRCS = src/store/store.c
182183
CYPHER_SRCS = src/cypher/cypher.c
183184

184185
# MCP server module (new)
185-
MCP_SRCS = src/mcp/mcp.c
186+
MCP_SRCS = src/mcp/mcp.c src/mcp/index_supervisor.c
186187

187188
# Discover module (new)
188189
DISCOVER_SRCS = \
@@ -319,7 +320,8 @@ TEST_FOUNDATION_SRCS = \
319320
tests/test_log.c \
320321
tests/test_str_util.c \
321322
tests/test_platform.c \
322-
tests/test_dump_verify.c
323+
tests/test_dump_verify.c \
324+
tests/test_subprocess.c
323325

324326
TEST_EXTRACTION_SRCS = \
325327
tests/test_extraction.c \

‎internal/cbm/cbm.c‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
#include "lsp/rust_lsp.h"
1515
#include "preprocessor.h"
1616
#include "foundation/compat.h"
17+
#include "foundation/compat_fs.h" // cbm_fopen — crash-supervisor per-file marker write
18+
#include "foundation/hash_table.h" // CBMHashTable — crash-supervisor quarantine set
1719
#include "tree_sitter/api.h" // TSParser, TSNode, TSTree, TSInput, TSLanguage, TSPoint, TSParseOptions, TSParseState
1820
#include "foundation/constants.h"
1921
#include "mimalloc.h" // mi_malloc/mi_calloc/mi_realloc/mi_free/mi_usable_size — bind 3rd-party allocators (#424)
@@ -566,6 +568,146 @@ static int count_params_from_signature(const char *sig) {
566568

567569
// --- Main extraction function ---
568570

571+
/* Test-only deterministic fault injection for the crash/hang supervisor tests.
572+
* Gated entirely behind env vars that are never set in production; a matching
573+
* rel_path either aborts (a fault signal the supervisor classifies as a crash)
574+
* or spins forever (an external-scanner infinite loop the quiet-timeout kills).
575+
* This gives an honest guard — green iff the supervisor actually contains a real
576+
* fault — instead of a fixture that may stop faulting once a root cause is fixed. */
577+
/* Crash-supervisor per-file marker (Stage 3c skip-and-continue). In the
578+
* supervisor's single-threaded recovery re-run, cbm_extract_file records the
579+
* file it is ABOUT to process here (CBM_INDEX_MARKER_FILE) before touching it,
580+
* so that if this file hard-crashes the worker the parent can read the marker
581+
* back and learn the EXACT crasher to quarantine. Only meaningful single-
582+
* threaded (parallel would race the marker); the env var is set solely by the
583+
* supervisor during recovery, so it is a no-op on every normal run. */
584+
static void cbm_index_write_marker(const char *rel_path) {
585+
const char *mf = getenv("CBM_INDEX_MARKER_FILE");
586+
if (!mf || !mf[0] || !rel_path || !rel_path[0]) {
587+
return;
588+
}
589+
FILE *f = cbm_fopen(mf, "wb");
590+
if (f) {
591+
(void)fputs(rel_path, f);
592+
(void)fclose(f);
593+
}
594+
}
595+
596+
/* ── Crash-quarantine set (Stage 3c skip-and-continue) ──────────────────────
597+
* After a crash the supervisor re-runs the worker single-threaded, passing
598+
* CBM_INDEX_QUARANTINE_FILE — a newline-delimited list of repo-relative paths
599+
* that already crashed the indexer and MUST NOT be extracted again. Owned here,
600+
* next to the other env-driven extract hooks (marker + fault injector), so the
601+
* single hard guard lives at the one choke point every pass funnels through
602+
* (cbm_extract_file): whether a pass re-extracts from disk on a cache miss
603+
* (sequential pass_calls/usages/semantic) or extracts fresh, a quarantined file
604+
* short-circuits to an empty result and never reaches the parser/crash. The
605+
* pipeline extract loops separately REPORT the skip as phase="crash" via
606+
* cbm_index_is_quarantined() so the crasher surfaces in the response skipped[].
607+
* Loaded once, lazily; read-only after load (safe for the parallel workers,
608+
* though recovery runs single-threaded). Unset env ⇒ empty set ⇒ cheap no-op. */
609+
static CBMHashTable *g_quarantine_set = NULL;
610+
enum { CBM_QSET_UNINIT = 0, CBM_QSET_INITING = 1, CBM_QSET_INITED = 2 };
611+
static atomic_int g_quarantine_state = CBM_QSET_UNINIT;
612+
613+
static void cbm_quarantine_load(void) {
614+
const char *qf = getenv("CBM_INDEX_QUARANTINE_FILE");
615+
if (!qf || !qf[0]) {
616+
return; /* normal path: empty set */
617+
}
618+
FILE *f = cbm_fopen(qf, "rb");
619+
if (!f) {
620+
return;
621+
}
622+
CBMHashTable *set = cbm_ht_create(16);
623+
if (!set) {
624+
(void)fclose(f);
625+
return;
626+
}
627+
char line[2048];
628+
while (fgets(line, sizeof(line), f)) {
629+
size_t len = strlen(line);
630+
while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) {
631+
line[--len] = '\0';
632+
}
633+
if (len == 0) {
634+
continue;
635+
}
636+
/* Line format: "path\tphase" where phase is "crash" or "hang". A bare
637+
* "path" line (no tab) is tolerated and defaults to phase "crash" for
638+
* backward compatibility with older quarantine files. */
639+
char *tab = strchr(line, '\t');
640+
const char *phase = "crash";
641+
if (tab) {
642+
*tab = '\0';
643+
if (tab[1]) {
644+
phase = tab + 1;
645+
}
646+
}
647+
if (line[0] == '\0') {
648+
continue; /* empty path (line began with a tab) — skip */
649+
}
650+
/* The table borrows the key + value pointers, so dup both. Intentionally
651+
* never freed: the set lives for the whole (short-lived worker) process.
652+
* The value stores the phase so cbm_index_quarantine_phase() can report
653+
* "crash" vs "hang"; membership (cbm_index_is_quarantined) is value != NULL. */
654+
char *key = cbm_strdup(line);
655+
char *pval = cbm_strdup(phase);
656+
if (key && pval) {
657+
cbm_ht_set(set, key, (void *)pval);
658+
}
659+
}
660+
(void)fclose(f);
661+
g_quarantine_set = set;
662+
}
663+
664+
bool cbm_index_is_quarantined(const char *rel_path) {
665+
if (!rel_path || !rel_path[0]) {
666+
return false;
667+
}
668+
int state = atomic_load(&g_quarantine_state);
669+
if (state != CBM_QSET_INITED) {
670+
/* First caller wins the CAS and loads; racers spin until INITED.
671+
* Same once-init pattern as cbm_ui_log_init (http_server.c). */
672+
state = CBM_QSET_UNINIT;
673+
if (atomic_compare_exchange_strong(&g_quarantine_state, &state, CBM_QSET_INITING)) {
674+
cbm_quarantine_load();
675+
atomic_store(&g_quarantine_state, CBM_QSET_INITED);
676+
} else {
677+
while (atomic_load(&g_quarantine_state) != CBM_QSET_INITED) {
678+
cbm_usleep(1000); /* 1ms */
679+
}
680+
}
681+
}
682+
return g_quarantine_set && cbm_ht_has(g_quarantine_set, rel_path);
683+
}
684+
685+
const char *cbm_index_quarantine_phase(const char *rel_path) {
686+
/* cbm_index_is_quarantined drives the lazy once-load and returns true only
687+
* when the set is loaded and holds rel_path — so on true, g_quarantine_set is
688+
* non-NULL and the stored value is the phase string ("crash"/"hang"). */
689+
if (!cbm_index_is_quarantined(rel_path)) {
690+
return NULL;
691+
}
692+
return (const char *)cbm_ht_get(g_quarantine_set, rel_path);
693+
}
694+
695+
static void cbm_test_fault_inject(const char *rel_path) {
696+
if (!rel_path || !rel_path[0]) {
697+
return;
698+
}
699+
const char *crash_on = getenv("CBM_TEST_CRASH_ON");
700+
if (crash_on && crash_on[0] && strstr(rel_path, crash_on)) {
701+
abort(); /* SIGABRT → WIFSIGNALED → classified as a crash */
702+
}
703+
const char *hang_on = getenv("CBM_TEST_HANG_ON");
704+
if (hang_on && hang_on[0] && strstr(rel_path, hang_on)) {
705+
for (;;) {
706+
/* Busy-spin: the supervisor's quiet-timeout kills + reports us. */
707+
}
708+
}
709+
}
710+
569711
CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage language,
570712
const char *project, const char *rel_path, int64_t timeout_micros,
571713
const char **extra_defines, const char **include_paths) {
@@ -579,6 +721,20 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage
579721
cbm_arena_init(&result->arena);
580722
CBMArena *a = &result->arena;
581723

724+
/* Crash-quarantine hard guard (Stage 3c): a file the supervisor pinned as a
725+
* crasher must NEVER be parsed again. Return a clean empty result BEFORE the
726+
* marker write and fault injector so no pass (including sequential re-extract
727+
* passes that miss the result cache) can crash on it. The pipeline extract
728+
* loops separately record it as a phase="crash" skip. Checked before the
729+
* marker so quarantined files never overwrite it — the marker keeps pointing
730+
* at the real (non-quarantined) file being processed when a crash hits. */
731+
if (cbm_index_is_quarantined(rel_path)) {
732+
return result;
733+
}
734+
735+
cbm_index_write_marker(rel_path);
736+
cbm_test_fault_inject(rel_path);
737+
582738
// Get language spec
583739
const CBMLangSpec *spec = cbm_lang_spec(language);
584740
if (!spec) {

‎internal/cbm/cbm.h‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -516,6 +516,20 @@ void cbm_alloc_init(void);
516516
// Initialize the library. Call once at startup. Returns 0 on success.
517517
int cbm_init(void);
518518

519+
// True when rel_path is in the crash-quarantine set — the newline-delimited list
520+
// of files (CBM_INDEX_QUARANTINE_FILE) the crash supervisor pinned as crashers
521+
// during its single-threaded recovery re-run. Loaded once, lazily; read-only
522+
// after load. cbm_extract_file short-circuits such files to an empty result so no
523+
// pass can crash on them; the pipeline extract loops call this to also REPORT the
524+
// skip as phase="crash". Always false (cheap no-op) when the env var is unset.
525+
bool cbm_index_is_quarantined(const char *rel_path);
526+
527+
// Phase a quarantined file was pinned under: "crash" (a fault signal) or "hang"
528+
// (killed for making no progress). Returns NULL when rel_path is not quarantined.
529+
// Drives the same lazy once-load as cbm_index_is_quarantined. Used by the pipeline
530+
// extract loops to report the skip's phase in skipped[] (falls back to "crash").
531+
const char *cbm_index_quarantine_phase(const char *rel_path);
532+
519533
// Extract all data from one file. Caller must call cbm_free_result().
520534
// source must remain valid for the duration of the call.
521535
// timeout_micros: per-file parse timeout in microseconds (0 = no timeout).

‎scripts/security-allowlist.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
src/foundation/compat_fs.c:popen:cbm_popen wrapper definition (POSIX)
99
src/foundation/compat_fs.c:cbm_popen:cbm_popen function definition
1010
src/foundation/compat_fs.c:fork:cbm_exec_no_shell — fork+execvp for shell-free subprocess execution
11+
src/foundation/subprocess.c:fork:cbm_run_posix — fork+execv for the crash/hang-isolating index worker (supervisor primitive; child execs immediately, no code runs in the forked image)
1112
src/foundation/compat_fs.c:execvp:cbm_exec_no_shell — direct exec without shell interpretation
1213

1314
# ── CLI: update command (user-initiated, interactive) ──────────────────────

0 commit comments

Comments
 (0)