Skip to content

Commit 21591d5

Browse files
DeusDataclaude
andcommitted
fix(daemon): slice the worker memory budget by active jobs, not by the job limit
The daemon handed every index worker aggregate/physical_job_limit (a fixed quarter by default) regardless of how many jobs were running, so a lone worker on a host like DeusData#1864 was capped at a quarter of the memory the daemon itself was allowed — and with the budget now failing whole (decision A) that fixed slice would have turned an idle host into a guaranteed over_memory_budget. Decision 2 (DeusData#1997, DeusData#832): the slice is decided at spawn time as aggregate / jobs active right then (the job being spawned included), so a lone worker may use the whole aggregate and concurrent jobs split it. The explicit CBM_MEM_BUDGET_MB override keeps its precedence inside the worker (bf46b91); the sub-byte-per-slot guard is unchanged. The spawn line logs project, active_jobs, aggregate_mb and slice_mb so a fail-whole verdict can be read against the budget the worker really had. Tests: sequenced admission proves first=aggregate, second=aggregate/2; the racing 4-way admission test accepts any aggregate/1..4 slice (the divisor is scheduling-dependent there); a scripted CLEAN worker returning the over_memory_budget error response is passed through verbatim on the first attempt — no recovery loop, no quarantine. Refs DeusData#1997 DeusData#832 DeusData#1864 Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent d72250a commit 21591d5

2 files changed

Lines changed: 205 additions & 17 deletions

File tree

‎src/daemon/application.c‎

Lines changed: 54 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ struct cbm_daemon_application {
181181
cbm_daemon_application_update_ops_t update_ops;
182182
cbm_project_lock_manager_t *project_locks;
183183
size_t physical_job_limit;
184-
size_t worker_memory_budget_bytes;
184+
size_t aggregate_memory_budget_bytes;
185185
size_t active_mutations;
186186
size_t update_owners;
187187
cbm_daemon_application_update_worker_t update_worker;
@@ -693,6 +693,8 @@ static bool application_truncate_file(const char *path) {
693693
}
694694

695695
static bool application_job_cancel_requested(cbm_daemon_application_job_t *job);
696+
static size_t application_worker_memory_slice_locked(cbm_daemon_application_t *application,
697+
size_t *active_jobs_out);
696698

697699
static char **application_read_suspects(cbm_daemon_application_job_t *job, const char *path,
698700
int *count_out, bool *cancelled_out) {
@@ -1088,11 +1090,33 @@ static application_attempt_status_t application_job_run_attempt(cbm_daemon_appli
10881090
return APPLICATION_ATTEMPT_CANCELLED;
10891091
}
10901092

1093+
/* Decision 2 (#1997 #832): the slice is decided at spawn time from the jobs
1094+
* active right now (this job included), so a lone worker may use the whole
1095+
* aggregate and concurrent jobs split it. The divisor is logged so a
1096+
* fail-whole verdict can be read against the budget the worker really had. */
1097+
size_t active_jobs = 0;
1098+
cbm_mutex_lock(&application->mutex);
1099+
size_t memory_budget_bytes = application_worker_memory_slice_locked(application, &active_jobs);
1100+
size_t aggregate_memory_budget_bytes = application->aggregate_memory_budget_bytes;
1101+
cbm_mutex_unlock(&application->mutex);
1102+
if (memory_budget_bytes > 0) {
1103+
char active_text[32];
1104+
char aggregate_text[32];
1105+
char slice_text[32];
1106+
(void)snprintf(active_text, sizeof(active_text), "%zu", active_jobs);
1107+
(void)snprintf(aggregate_text, sizeof(aggregate_text), "%zu",
1108+
aggregate_memory_budget_bytes / (1024U * 1024U));
1109+
(void)snprintf(slice_text, sizeof(slice_text), "%zu",
1110+
memory_budget_bytes / (1024U * 1024U));
1111+
cbm_log_info("daemon.index.worker_budget", "project", job->project_key, "active_jobs",
1112+
active_text, "aggregate_mb", aggregate_text, "slice_mb", slice_text);
1113+
}
1114+
10911115
cbm_daemon_application_worker_t worker = NULL;
10921116
application_tmp_lock();
1093-
int start_result = application->worker_ops.start(
1094-
application->worker_ops.context, job->args_json, application->worker_memory_budget_bytes,
1095-
marker_path, quarantine_path, &worker);
1117+
int start_result =
1118+
application->worker_ops.start(application->worker_ops.context, job->args_json,
1119+
memory_budget_bytes, marker_path, quarantine_path, &worker);
10961120
application_tmp_unlock();
10971121
if (start_result != 0 || !worker) {
10981122
return application_job_cancel_requested(job) ? APPLICATION_ATTEMPT_CANCELLED
@@ -1564,6 +1588,22 @@ static size_t application_active_job_count_locked(cbm_daemon_application_t *appl
15641588
return count;
15651589
}
15661590

1591+
/* Decision 2 (#1997 #832): a worker's memory slice is the aggregate budget
1592+
* divided by the jobs active at spawn time. The job being spawned is already
1593+
* in the table, so the divisor never drops below one; a zero aggregate means
1594+
* "no cap" and the worker falls back to its own RAM-fraction budget. */
1595+
static size_t application_worker_memory_slice_locked(cbm_daemon_application_t *application,
1596+
size_t *active_jobs_out) {
1597+
size_t active = application_active_job_count_locked(application);
1598+
if (active == 0) {
1599+
active = 1;
1600+
}
1601+
if (active_jobs_out) {
1602+
*active_jobs_out = active;
1603+
}
1604+
return application->aggregate_memory_budget_bytes / active;
1605+
}
1606+
15671607
/* Compare the effective index request, not its JSON spelling. yyjson's deep
15681608
* equality treats object member order as insignificant, while the small
15691609
* normalization below removes values that the index handler interprets as
@@ -2948,18 +2988,19 @@ cbm_daemon_application_t *cbm_daemon_application_new(
29482988
application->ui_readiness_secret_set = true;
29492989
}
29502990
}
2951-
/* Equal fixed slices keep admission deterministic: starting fewer jobs does
2952-
* not let an early worker claim memory reserved for later concurrent jobs.
2953-
* The absurd sub-byte-per-slot case is made safe by reducing effective
2954-
* capacity before division; normal daemon budgets are many orders larger. */
2991+
/* The per-worker slice is decided at spawn time (see
2992+
* application_worker_memory_slice_locked): the aggregate divided by the
2993+
* jobs active then, so a lone worker may use the whole aggregate and
2994+
* concurrent jobs split it (decision 2, #1997 #832) — the former fixed
2995+
* aggregate/limit slice starved a lone job on hosts like #1864. The absurd
2996+
* sub-byte-per-slot case is still made safe by reducing effective capacity
2997+
* so every admitted job can receive at least one byte; normal daemon
2998+
* budgets are many orders larger. */
29552999
if (aggregate_memory_budget_bytes > 0 &&
29563000
application->physical_job_limit > aggregate_memory_budget_bytes) {
29573001
application->physical_job_limit = aggregate_memory_budget_bytes;
29583002
}
2959-
if (aggregate_memory_budget_bytes > 0 && application->physical_job_limit > 0) {
2960-
application->worker_memory_budget_bytes =
2961-
aggregate_memory_budget_bytes / application->physical_job_limit;
2962-
}
3003+
application->aggregate_memory_budget_bytes = aggregate_memory_budget_bytes;
29633004
if (!application->worker_ops.start) {
29643005
application->worker_ops = (cbm_daemon_application_worker_ops_t){
29653006
.context = NULL,
@@ -3558,7 +3599,7 @@ size_t cbm_daemon_application_worker_memory_budget_bytes(cbm_daemon_application_
35583599
return 0;
35593600
}
35603601
cbm_mutex_lock(&application->mutex);
3561-
size_t budget = application->worker_memory_budget_bytes;
3602+
size_t budget = application_worker_memory_slice_locked(application, NULL);
35623603
cbm_mutex_unlock(&application->mutex);
35633604
return budget;
35643605
}

‎tests/test_daemon_application.c‎

Lines changed: 151 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5245,12 +5245,157 @@ TEST(daemon_application_default_limit_admits_four_and_rejects_fifth) {
52455245
ASSERT_EQ(atomic_load(&fake.starts), DEFAULT_CAP_RUNNING);
52465246
ASSERT_EQ(atomic_load(&fake.cancels), DEFAULT_CAP_RUNNING);
52475247
ASSERT_EQ(atomic_load(&fake.destroys), DEFAULT_CAP_RUNNING);
5248-
size_t assigned_budget = 0;
5248+
/* Decision 2 (#1997 #832): each worker's slice is the aggregate divided by
5249+
* the jobs active when IT was spawned. Four requests race through
5250+
* admission, so which divisor each saw is scheduling-dependent — but every
5251+
* slice is one of aggregate/1..4 and never exceeds the aggregate. */
52495252
for (int i = 0; i < DEFAULT_CAP_RUNNING; i++) {
5250-
ASSERT_EQ(fake.memory_budgets[i], aggregate_budget / DEFAULT_CAP_RUNNING);
5251-
assigned_budget += fake.memory_budgets[i];
5253+
bool valid_slice = false;
5254+
for (size_t divisor = 1; divisor <= DEFAULT_CAP_RUNNING; divisor++) {
5255+
valid_slice = valid_slice || fake.memory_budgets[i] == aggregate_budget / divisor;
5256+
}
5257+
ASSERT_TRUE(valid_slice);
5258+
ASSERT_TRUE(fake.memory_budgets[i] <= aggregate_budget);
5259+
}
5260+
PASS();
5261+
}
5262+
5263+
/* Decision 2 (#1997 #832): the worker slice is aggregate / jobs ACTIVE at
5264+
* spawn time. Sequenced admission makes the divisor deterministic: the first
5265+
* job spawns alone and receives the whole aggregate; the second spawns while
5266+
* the first still runs and receives half. */
5267+
TEST(daemon_application_worker_slice_is_aggregate_over_active_jobs) {
5268+
const size_t aggregate_budget = 4099;
5269+
app_fake_worker_context_t fake;
5270+
app_fake_worker_context_init(&fake);
5271+
cbm_daemon_application_worker_ops_t worker_ops = {
5272+
.context = &fake,
5273+
.start = app_fake_worker_start,
5274+
.poll = app_fake_worker_poll,
5275+
.cancel = app_fake_worker_cancel,
5276+
.log_path = app_fake_worker_log_path,
5277+
.destroy = app_fake_worker_destroy,
5278+
};
5279+
cbm_daemon_application_config_t config = {
5280+
.worker_ops = &worker_ops,
5281+
.aggregate_memory_budget_bytes = aggregate_budget,
5282+
};
5283+
cbm_daemon_application_t *application = cbm_daemon_application_new(&config);
5284+
char roots[2][APP_TEST_PATH_CAP];
5285+
bool roots_ok = true;
5286+
for (int i = 0; i < 2; i++) {
5287+
(void)snprintf(roots[i], sizeof(roots[i]), "%s/cbm-app-slice-%d-XXXXXX", cbm_tmpdir(), i);
5288+
roots_ok = roots_ok && cbm_mkdtemp(roots[i]) != NULL;
5289+
}
5290+
app_index_thread_t requests[2] = {
5291+
{.application = application, .project = "slice-first", .root = roots[0], .result = -1},
5292+
{.application = application, .project = "slice-second", .root = roots[1], .result = -1},
5293+
};
5294+
cbm_thread_t threads[2];
5295+
bool first_started = application && roots_ok &&
5296+
cbm_thread_create(&threads[0], 0, app_index_thread, &requests[0]) == 0;
5297+
bool first_spawned = first_started && app_wait_for_atomic_int(&fake.starts, 1);
5298+
bool second_started =
5299+
first_spawned && cbm_thread_create(&threads[1], 0, app_index_thread, &requests[1]) == 0;
5300+
bool second_spawned = second_started && app_wait_for_atomic_int(&fake.starts, 2);
5301+
atomic_store(&fake.allow_completion, true);
5302+
if (first_started) {
5303+
(void)cbm_thread_join(&threads[0]);
5304+
}
5305+
if (second_started) {
5306+
(void)cbm_thread_join(&threads[1]);
5307+
}
5308+
bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS);
5309+
cbm_daemon_application_free(application);
5310+
for (int i = 0; i < 2; i++) {
5311+
(void)cbm_rmdir(roots[i]);
5312+
}
5313+
5314+
ASSERT_TRUE(roots_ok);
5315+
ASSERT_TRUE(first_spawned);
5316+
ASSERT_TRUE(second_spawned);
5317+
ASSERT_EQ(requests[0].result, 0);
5318+
ASSERT_EQ(requests[1].result, 0);
5319+
ASSERT_TRUE(stopped);
5320+
ASSERT_EQ(fake.memory_budgets[0], aggregate_budget);
5321+
ASSERT_EQ(fake.memory_budgets[1], aggregate_budget / 2);
5322+
PASS();
5323+
}
5324+
5325+
/* Decision A (#1997 #832): the over-budget verdict travels as a CLEAN worker
5326+
* exit carrying an error response. The daemon passes that response through
5327+
* verbatim on the first attempt — no crash/hang recovery loop, no quarantine
5328+
* of innocent files — because a healthy process reported an honest failure. */
5329+
TEST(daemon_application_over_budget_response_passes_through_without_recovery) {
5330+
static const char over_budget_response[] =
5331+
"{\"content\":[{\"type\":\"text\",\"text\":\"{\\\"project\\\":\\\"budget\\\","
5332+
"\\\"status\\\":\\\"error\\\",\\\"reason\\\":\\\"over_memory_budget\\\","
5333+
"\\\"previous_index\\\":\\\"preserved\\\"}\"}],\"isError\":true}";
5334+
app_fake_worker_context_t fake;
5335+
app_fake_worker_context_init(&fake);
5336+
atomic_store(&fake.scripted, true);
5337+
fake.outcomes[0] = CBM_PROC_CLEAN;
5338+
fake.responses[0] = over_budget_response;
5339+
fake.outcomes[1] = CBM_PROC_CLEAN; /* a second start would be the recovery loop */
5340+
cbm_daemon_application_worker_ops_t worker_ops = {
5341+
.context = &fake,
5342+
.start = app_fake_worker_start,
5343+
.poll = app_fake_worker_poll,
5344+
.cancel = app_fake_worker_cancel,
5345+
.log_path = app_fake_worker_log_path,
5346+
.destroy = app_fake_worker_destroy,
5347+
};
5348+
cbm_daemon_application_config_t config = {.worker_ops = &worker_ops};
5349+
cbm_daemon_application_t *application = cbm_daemon_application_new(&config);
5350+
cbm_daemon_runtime_application_callbacks_t callbacks =
5351+
cbm_daemon_application_runtime_callbacks(application);
5352+
cbm_daemon_runtime_application_session_t *session =
5353+
application ? app_test_open(&callbacks, 71) : NULL;
5354+
char root[APP_TEST_PATH_CAP];
5355+
snprintf(root, sizeof(root), "%s/cbm-app-over-budget-XXXXXX", cbm_tmpdir());
5356+
bool root_ok = cbm_mkdtemp(root) != NULL;
5357+
uint8_t *context = NULL;
5358+
uint32_t context_length = 0;
5359+
char args[APP_TEST_PATH_CAP + 32];
5360+
snprintf(args, sizeof(args), "{\"repo_path\":\"%s\"}", root);
5361+
uint8_t *tool = NULL;
5362+
uint32_t tool_length = 0;
5363+
bool setup = application && session && root_ok &&
5364+
app_test_context_request(root, root, &context, &context_length) &&
5365+
app_test_tool_request("index_repository", args, &tool, &tool_length);
5366+
uint8_t *response = NULL;
5367+
uint32_t response_length = 0;
5368+
if (setup) {
5369+
setup = app_test_request(&callbacks, session, context, context_length, &response,
5370+
&response_length) == CBM_DAEMON_RUNTIME_APPLICATION_OK;
5371+
free(response);
5372+
response = NULL;
5373+
response_length = 0;
52525374
}
5253-
ASSERT_TRUE(assigned_budget <= aggregate_budget);
5375+
cbm_daemon_runtime_application_status_t status =
5376+
setup
5377+
? app_test_request(&callbacks, session, tool, tool_length, &response, &response_length)
5378+
: CBM_DAEMON_RUNTIME_APPLICATION_HANDLER_ERROR;
5379+
bool passed_through = response && response_length > 0 &&
5380+
strstr((char *)response, "over_memory_budget") != NULL &&
5381+
strstr((char *)response, "\"isError\":true") != NULL;
5382+
free(response);
5383+
free(context);
5384+
free(tool);
5385+
if (session) {
5386+
callbacks.session_cancel(callbacks.context, session);
5387+
callbacks.session_close(callbacks.context, session);
5388+
}
5389+
bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS);
5390+
cbm_daemon_application_free(application);
5391+
(void)cbm_rmdir(root);
5392+
5393+
ASSERT_TRUE(setup);
5394+
ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_OK);
5395+
ASSERT_TRUE(passed_through);
5396+
ASSERT_EQ(atomic_load(&fake.starts), 1);
5397+
ASSERT_EQ(atomic_load(&fake.destroys), 1);
5398+
ASSERT_TRUE(stopped);
52545399
PASS();
52555400
}
52565401

@@ -5458,6 +5603,8 @@ SUITE(daemon_application) {
54585603
RUN_TEST(daemon_application_thread_start_failure_rolls_back_job_reservation);
54595604
RUN_TEST(daemon_application_queues_explicit_index_behind_physical_job_limit);
54605605
RUN_TEST(daemon_application_default_limit_admits_four_and_rejects_fifth);
5606+
RUN_TEST(daemon_application_worker_slice_is_aggregate_over_active_jobs);
5607+
RUN_TEST(daemon_application_over_budget_response_passes_through_without_recovery);
54615608
RUN_TEST(daemon_application_free_reports_retained_live_ownership);
54625609
RUN_TEST(daemon_application_rejects_clean_exit_when_process_tree_is_not_contained);
54635610
}

0 commit comments

Comments
 (0)