Skip to content

Commit 699be6c

Browse files
DeusDataclaude
andcommitted
fix(pipeline): fail an over-budget index attempt whole instead of soft-overshooting
The extraction memory gate was advisory: when a full back-pressure cycle ended still over budget it latched "futile" and kept admitting files, so a host that could not hold the graph ran on until the kernel killed the worker (DeusData#1997, DeusData#832) and the supervisor then quarantined innocent files. Now the worker that latches futility runs ONE confirmation cycle (collect + bounded naps). If resident memory drained, the gate re-arms as before. If it is still over budget, the attempt fails whole: every worker stops pulling, extract returns the new CBM_PIPELINE_ABORT_OVER_BUDGET, the orchestrator discards the staging DB on that non-zero code like any other, and the previously serving generation is left byte-identical. The verdict is a plain return value so it travels as a clean-exit worker response, never a kill. The abort flag is its own atomic, not the shared cancel token, which belongs to client cancellation. The abort/persist codes move to pipeline.h alongside the new one (callers legitimately distinguish them); FORCE_FULL_REINDEX stays internal. Tests: the futility test now asserts the new code (the confirmation adds one cycle to its bound), and a new test proves fail-whole preserves the live DB byte-for-byte, keeps its node count, and leaves no stage residue. Refs DeusData#1997 DeusData#832 DeusData#1973 DeusData#1864 Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 29dbf0b commit 699be6c

4 files changed

Lines changed: 251 additions & 36 deletions

File tree

‎src/pipeline/pass_parallel.c‎

Lines changed: 71 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@ enum {
3030
/* Extraction memory back-pressure: when the process is over its RSS budget,
3131
* a worker reclaims + naps before pulling another file so peers can finish
3232
* and return pages. Bounded spins avoid deadlock when the resident graph
33-
* itself is near budget (then proceed with a soft overshoot). */
33+
* itself is near budget: then ONE confirmation cycle decides — drained
34+
* means continue, still over means the attempt fails whole (decision A,
35+
* #1997 #832) instead of the former open-ended soft overshoot. */
3436
PP_BACKPRESSURE_MAX_SPINS = 40,
3537
PP_BACKPRESSURE_NAP_NS = 3000000, /* 3 ms */
3638
};
@@ -667,6 +669,11 @@ typedef struct {
667669
* While set, pulls skip the nap (the designed soft overshoot); the cheap
668670
* over-budget probe re-arms the gate once RSS drains under budget. */
669671
_Atomic int bp_futile;
672+
/* Decision A (#1997 #832): set once futility was CONFIRMED by a second
673+
* full cycle that still ended over budget. Every pull loop breaks on it
674+
* and extract returns CBM_PIPELINE_ABORT_OVER_BUDGET. Deliberately not
675+
* the shared cancel token — that one belongs to client cancellation. */
676+
_Atomic int over_budget_abort;
670677

671678
const CBMMacroTable *macro_table; /* ObjectScript $$$macros (NULL if none) */
672679
const CBMReturnTypeTable *return_type_table; /* ObjectScript return types (NULL if none) */
@@ -722,6 +729,39 @@ static void log_extract_done(int pos, uint64_t ms, int defs, const char *path) {
722729
}
723730
}
724731

732+
/* One back-pressure cycle: reclaim this thread's freed pages, then nap in
733+
* bounded 3 ms steps while the process stays over budget. Counted for test
734+
* observability. Returns true when the FULL cycle elapsed still over budget —
735+
* the resident floor, not in-flight transients, holds the memory. */
736+
static bool pp_backpressure_cycle_still_over(extract_ctx_t *ec) {
737+
cbm_mem_collect();
738+
atomic_fetch_add_explicit(&g_bp_nap_cycles, SKIP_ONE, memory_order_relaxed);
739+
int bp = 0;
740+
for (; bp < PP_BACKPRESSURE_MAX_SPINS && cbm_mem_over_budget() &&
741+
!atomic_load_explicit(ec->cancelled, memory_order_relaxed) &&
742+
!atomic_load_explicit(&ec->over_budget_abort, memory_order_relaxed);
743+
bp++) {
744+
struct timespec nap = {0, PP_BACKPRESSURE_NAP_NS};
745+
cbm_nanosleep(&nap, NULL);
746+
}
747+
return bp == PP_BACKPRESSURE_MAX_SPINS && cbm_mem_over_budget();
748+
}
749+
750+
/* Decision A (#1997 #832): back-pressure was futile and the confirmation
751+
* cycle still ended over budget. The budget stops being advisory here: flag
752+
* the attempt so every worker stops pulling and extract returns
753+
* CBM_PIPELINE_ABORT_OVER_BUDGET. Cooperative on purpose — the failure must
754+
* travel as a clean-exit worker response, never a kill, or the supervisor
755+
* would quarantine innocent files. */
756+
static void pp_fail_whole_over_budget(extract_ctx_t *ec) {
757+
if (atomic_exchange_explicit(&ec->over_budget_abort, 1, memory_order_relaxed) == 0) {
758+
cbm_log_error("mem.budget.exceeded", "rss_mb",
759+
itoa_log((int)(cbm_mem_rss() / (1024 * 1024))), "budget_mb",
760+
itoa_log((int)(cbm_mem_budget() / (1024 * 1024))), "phase",
761+
"parallel_extract", "action", "fail_whole");
762+
}
763+
}
764+
725765
static void extract_worker(int worker_id, void *ctx_ptr) {
726766
extract_ctx_t *ec = ctx_ptr;
727767
extract_worker_state_t *ws = &ec->workers[worker_id];
@@ -739,7 +779,8 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
739779
break;
740780
}
741781
cbm_scale_tick(&ec->scale, sort_pos);
742-
if (atomic_load_explicit(ec->cancelled, memory_order_relaxed)) {
782+
if (atomic_load_explicit(ec->cancelled, memory_order_relaxed) ||
783+
atomic_load_explicit(&ec->over_budget_abort, memory_order_relaxed)) {
743784
break;
744785
}
745786

@@ -750,39 +791,41 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
750791
* near the budget instead of letting all workers parse their biggest
751792
* files at once. Self-disabling when the budget is unset (tests) or RSS
752793
* is under budget; bounded spins avoid deadlock when the resident graph
753-
* is itself near budget (then proceed with a soft overshoot).
794+
* is itself near budget.
754795
*
755796
* Futility latch: when a FULL nap cycle ends still over budget, the
756797
* resident floor — not transients — holds the memory; napping again on
757798
* the next pull cannot reclaim it and only idles workers (linux kernel:
758-
* one full cycle per pull ≈ 390 s at 79% avg CPU). Latch bp_futile and
759-
* proceed with the soft overshoot; the over-budget probe below re-arms
760-
* the gate as soon as RSS drains under budget. */
799+
* one full cycle per pull ≈ 390 s at 79% avg CPU). The worker that
800+
* latches bp_futile (0→1) runs ONE confirmation cycle while its peers
801+
* skip the nap. A confirmation that ends under budget re-arms the gate
802+
* (RSS drained after all); one that still ends over budget fails the
803+
* attempt whole (decision A, #1997 #832): every worker stops pulling,
804+
* extract returns CBM_PIPELINE_ABORT_OVER_BUDGET, nothing is published
805+
* and the previously serving index keeps answering. */
761806
if (cbm_mem_budget() > 0) {
762807
bool over = cbm_mem_over_budget();
763808
bool futile = atomic_load_explicit(&ec->bp_futile, memory_order_relaxed) != 0;
764809
if (over && !futile) {
765-
cbm_mem_collect();
766-
atomic_fetch_add_explicit(&g_bp_nap_cycles, SKIP_ONE, memory_order_relaxed);
767-
int bp = 0;
768-
for (; bp < PP_BACKPRESSURE_MAX_SPINS && cbm_mem_over_budget() &&
769-
!atomic_load_explicit(ec->cancelled, memory_order_relaxed);
770-
bp++) {
771-
struct timespec nap = {0, PP_BACKPRESSURE_NAP_NS};
772-
cbm_nanosleep(&nap, NULL);
773-
}
774-
if (bp == PP_BACKPRESSURE_MAX_SPINS && cbm_mem_over_budget()) {
775-
/* Log only the 0→1 transition: all workers race into the
776-
* gate before anyone latches, so a plain store would WARN
777-
* once per worker (12 lines per latch event). */
778-
if (atomic_exchange_explicit(&ec->bp_futile, 1, memory_order_relaxed) == 0) {
779-
cbm_log_warn("mem.backpressure.futile", "action", "soft_overshoot");
810+
/* Act only on the 0→1 transition: all workers race into the
811+
* gate before anyone latches, so a plain store would confirm
812+
* (and WARN) once per worker. */
813+
if (pp_backpressure_cycle_still_over(ec) &&
814+
atomic_exchange_explicit(&ec->bp_futile, 1, memory_order_relaxed) == 0) {
815+
cbm_log_warn("mem.backpressure.futile", "action", "confirm");
816+
if (pp_backpressure_cycle_still_over(ec)) {
817+
pp_fail_whole_over_budget(ec);
818+
} else {
819+
atomic_store_explicit(&ec->bp_futile, 0, memory_order_relaxed);
780820
}
781821
}
782822
} else if (!over && futile) {
783823
atomic_store_explicit(&ec->bp_futile, 0, memory_order_relaxed);
784824
}
785825
}
826+
if (atomic_load_explicit(&ec->over_budget_abort, memory_order_relaxed)) {
827+
break;
828+
}
786829

787830
int file_idx = ec->sorted[sort_pos].idx;
788831
const cbm_file_info_t *fi = &ec->files[file_idx];
@@ -1129,6 +1172,7 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file
11291172
atomic_init(&ec.retain_cap_warned, 0);
11301173
atomic_init(&ec.oversized_warned, 0);
11311174
atomic_init(&ec.bp_futile, 0);
1175+
atomic_init(&ec.over_budget_abort, 0);
11321176

11331177
/* Sub-phase: Dispatch workers (parse + extract per file, PARALLEL) */
11341178
CBM_PROF_START(t_dispatch);
@@ -1176,6 +1220,12 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file
11761220
free(sorted);
11771221
cbm_macro_table_free(pp_macro_table); /* ObjectScript macro table (NULL-safe) */
11781222

1223+
/* The over-budget verdict outranks the cancel sentinel: a caller must be
1224+
* able to name the cause, and the orchestrator discards the staging DB on
1225+
* every non-zero code alike (the live generation is never touched). */
1226+
if (atomic_load(&ec.over_budget_abort)) {
1227+
return CBM_PIPELINE_ABORT_OVER_BUDGET;
1228+
}
11791229
if (atomic_load(ctx->cancelled)) {
11801230
return CBM_NOT_FOUND;
11811231
}

‎src/pipeline/pipeline.h‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,14 @@ void cbm_pipeline_free(cbm_pipeline_t *p);
6363
* need to know whether the PREVIOUS generation survived can distinguish the
6464
* failures by value: the run publishes by renaming a fully validated staging
6565
* database over the destination, so every abort before that rename leaves the
66-
* existing database in place. Those codes (CBM_PIPELINE_ABORT_PRESERVE_DB and
67-
* CBM_PIPELINE_PERSIST_FAILED) are defined in pipeline_internal.h alongside the
68-
* stages that raise them. */
66+
* existing database in place. */
67+
#define CBM_PIPELINE_ABORT_PRESERVE_DB (-2) /* aborted pre-publication; previous DB intact */
68+
#define CBM_PIPELINE_PERSIST_FAILED (-4) /* staging/rollback failure during publication */
69+
/* Resident memory stayed above the budget after back-pressure and one
70+
* confirmation cycle: the run stops before publication, no partial graph is
71+
* written, the previous DB is intact (#1997 #832). Distinct from the cancel
72+
* sentinel so callers can name the cause instead of "pipeline failed". */
73+
#define CBM_PIPELINE_ABORT_OVER_BUDGET (-5)
6974
int cbm_pipeline_run(cbm_pipeline_t *p);
7075

7176
/* Request cancellation of a running pipeline (thread-safe). */

‎src/pipeline/pipeline_internal.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,12 @@
2929
/* Route node QN buffer size (must fit __route__METHOD__/full/url/path) */
3030
#define CBM_ROUTE_QN_SIZE 768
3131

32-
/* Incremental integrity failure: abort the run and preserve the existing DB.
33-
* Distinct from CBM_NOT_FOUND, which the orchestrator uses as the normal
32+
/* CBM_PIPELINE_ABORT_PRESERVE_DB / CBM_PIPELINE_PERSIST_FAILED moved to
33+
* pipeline.h — callers legitimately distinguish them (the header's contract
34+
* always said so). FORCE_FULL_REINDEX never escapes the orchestrator and
35+
* stays internal. All three are distinct from CBM_NOT_FOUND, the normal
3436
* "no incremental route; continue with a full index" sentinel. */
35-
#define CBM_PIPELINE_ABORT_PRESERVE_DB (-2)
3637
#define CBM_PIPELINE_FORCE_FULL_REINDEX (-3)
37-
#define CBM_PIPELINE_PERSIST_FAILED (-4)
3838

3939
/* Canonicalize route-path parameter placeholders (":id", "{id}", "<id>",
4040
* "${...}") to a single "{}" token so that client call sites and server

0 commit comments

Comments
 (0)