Skip to content

Commit 5a9f6e7

Browse files
authored
Merge pull request DeusData#2494 from DeusData/fix/issue-1300-silent-worker-exit
fix(index): report clean worker exits without a response
2 parents bad3e48 + 7acb5ea commit 5a9f6e7

11 files changed

Lines changed: 462 additions & 1 deletion

‎src/daemon/application.c‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1349,6 +1349,15 @@ static char *application_job_failure_response(const cbm_index_worker_result_t *r
13491349
(void)snprintf(message, sizeof(message),
13501350
"index worker containment failed (%s); inspect log: %s",
13511351
cbm_proc_outcome_str(result->outcome), log_path ? log_path : "unavailable");
1352+
} else if (result && result->response_missing) {
1353+
/* #1300: name the silent clean exit, where it stopped, and the log. */
1354+
const char *log = log_path ? log_path
1355+
: result->worker_log[0] ? result->worker_log
1356+
: "unavailable";
1357+
(void)snprintf(message, sizeof(message),
1358+
"index worker exited cleanly (exit 0) without writing a response; the "
1359+
"index was not published; last phase reached: %s; inspect log: %s",
1360+
result->last_phase[0] ? result->last_phase : "unknown", log);
13521361
} else if (result) {
13531362
(void)snprintf(message, sizeof(message),
13541363
"index worker ended with %s (exit=%d, signal=%d); inspect log: %s",

‎src/mcp/index_supervisor.c‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,7 @@ typedef enum {
359359
WORKER_RESPONSE_READ_OK = 0,
360360
WORKER_RESPONSE_READ_ERROR,
361361
WORKER_RESPONSE_READ_TOO_LARGE,
362+
WORKER_RESPONSE_READ_EMPTY, /* the pre-created file was never written (#1300) */
362363
} worker_response_read_status_t;
363364

364365
/* Read at most one daemon-application payload. The worker is already reaped,
@@ -384,6 +385,14 @@ static char *slurp_worker_response(const char *path, worker_response_read_status
384385
(void)fclose(f);
385386
return NULL;
386387
}
388+
if (n == 0) {
389+
/* The supervisor creates the response file before spawning, so an
390+
* empty file is a worker that never wrote its result. Reading it as
391+
* "" made that a successful run with an empty response (#1300). */
392+
*status_out = WORKER_RESPONSE_READ_EMPTY;
393+
(void)fclose(f);
394+
return NULL;
395+
}
387396
if (fseek(f, 0, SEEK_SET) != 0) {
388397
(void)fclose(f);
389398
return NULL;
@@ -405,6 +414,67 @@ static char *slurp_worker_response(const char *path, worker_response_read_status
405414
return buf;
406415
}
407416

417+
enum {
418+
WORKER_LOG_TAIL_BYTES = 64 * 1024, /* last-phase scan window: bounded, never the whole log */
419+
WORKER_LOG_LINE_CAP = 1024,
420+
};
421+
422+
/* Copy the event name of one structured log line ("level=... msg=<event> ..."
423+
* or {"level":...,"event":"<event>",...}) into out. False for any other line
424+
* (plain stderr text, a truncated fragment). */
425+
static bool worker_log_line_event(const char *line, char *out, size_t out_size) {
426+
static const char text_key[] = " msg=";
427+
static const char json_key[] = "\"event\":\"";
428+
const char *start = NULL;
429+
char stop = ' ';
430+
if (strncmp(line, "level=", strlen("level=")) == 0) {
431+
const char *msg = strstr(line, text_key);
432+
start = msg ? msg + strlen(text_key) : NULL;
433+
} else if (strncmp(line, "{\"level\":", strlen("{\"level\":")) == 0) {
434+
const char *event = strstr(line, json_key);
435+
start = event ? event + strlen(json_key) : NULL;
436+
stop = '"';
437+
}
438+
if (!start) {
439+
return false;
440+
}
441+
size_t length = 0;
442+
while (start[length] && start[length] != stop && start[length] != '\n' &&
443+
start[length] != '\r') {
444+
length++;
445+
}
446+
if (length == 0 || length >= out_size) {
447+
return false;
448+
}
449+
memcpy(out, start, length);
450+
out[length] = '\0';
451+
return true;
452+
}
453+
454+
/* #1300: the last structured event the worker logged before it exited — the
455+
* phase it reached. Scans only the log's final WORKER_LOG_TAIL_BYTES. */
456+
static void worker_log_last_phase(const char *log_path, char *out, size_t out_size) {
457+
(void)snprintf(out, out_size, "%s", "unknown");
458+
FILE *log = cbm_fopen(log_path, "rb");
459+
if (!log) {
460+
return;
461+
}
462+
if (fseek(log, 0, SEEK_END) == 0) {
463+
long size = ftell(log);
464+
long from = size > WORKER_LOG_TAIL_BYTES ? size - WORKER_LOG_TAIL_BYTES : 0;
465+
if (size >= 0 && fseek(log, from, SEEK_SET) == 0) {
466+
char line[WORKER_LOG_LINE_CAP];
467+
char event[CBM_SZ_128];
468+
while (fgets(line, sizeof(line), log)) {
469+
if (worker_log_line_event(line, event, sizeof(event))) {
470+
(void)snprintf(out, out_size, "%s", event);
471+
}
472+
}
473+
}
474+
}
475+
(void)fclose(log);
476+
}
477+
408478
enum {
409479
INDEX_WORKER_PATH_CAP = CBM_SZ_4K,
410480
INDEX_WORKER_ARGV_CAP = 17,
@@ -634,6 +704,10 @@ static void worker_terminal_log(cbm_index_worker_handle_t *handle) {
634704
} else if (handle->result.cancellation_requested) {
635705
cbm_log_warn("index.supervisor.worker_cancelled", "outcome",
636706
cbm_proc_outcome_str(handle->result.outcome), "log", handle->log_path);
707+
} else if (handle->result.response_missing) {
708+
/* #1300: keep and name the log — it is the only record of the run. */
709+
cbm_log_error("index.supervisor.no_response", "exit_code", exit_text, "last_phase",
710+
handle->result.last_phase, "log", handle->log_path);
637711
} else if (handle->result.outcome == CBM_PROC_CLEAN && !cbm_profile_active) {
638712
(void)cbm_unlink(handle->log_path);
639713
} else if (handle->result.outcome == CBM_PROC_CLEAN) {
@@ -828,6 +902,16 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle,
828902
handle->result.response_rejected = true;
829903
handle->result.outcome = CBM_PROC_EXIT_NONZERO;
830904
handle->result.exit_code = -1;
905+
} else if (!handle->result.response) {
906+
/* #1300: a clean exit that wrote no response. The outcome stays
907+
* CLEAN (the disposition is FALLBACK: an explicit error, no
908+
* in-process fallback and no recovery re-run of a worker that did
909+
* not crash); the log and the last phase it reached are kept. */
910+
handle->result.response_missing = true;
911+
worker_log_last_phase(handle->log_path, handle->result.last_phase,
912+
sizeof(handle->result.last_phase));
913+
(void)snprintf(handle->result.worker_log, sizeof(handle->result.worker_log), "%s",
914+
handle->log_path);
831915
}
832916
}
833917
(void)cbm_unlink(handle->response_path);

‎src/mcp/index_supervisor.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,14 @@ typedef struct {
147147
bool forced;
148148
bool tree_quiesced;
149149
bool supervision_failed;
150+
/* #1300: the worker exited CLEAN but wrote no response (an empty or
151+
* unreadable response file). A contained failure, never a success: the
152+
* response stays NULL, the worker log is retained and named, and
153+
* last_phase is the last structured event that log recorded ("unknown"
154+
* when it holds none), so the report says where the run stopped. */
155+
bool response_missing;
156+
char last_phase[128];
157+
char worker_log[4096];
150158
bool response_rejected; /* clean worker exceeded the bounded response protocol */
151159
char *response; /* worker result only after a contained, uncancelled CLEAN exit;
152160
* borrowed for async polls, caller-owned from the sync wrapper */

‎src/mcp/mcp.c‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10900,6 +10900,44 @@ int cbm_index_restart_cap_for_testing(void) {
1090010900
return index_restart_cap();
1090110901
}
1090210902

10903+
char *cbm_mcp_index_worker_no_response_failure(const char *args,
10904+
const cbm_index_worker_result_t *worker_result) {
10905+
const char *phase =
10906+
worker_result && worker_result->last_phase[0] ? worker_result->last_phase : "unknown";
10907+
const char *log =
10908+
worker_result && worker_result->worker_log[0] ? worker_result->worker_log : "unavailable";
10909+
/* Borrow repo_path from a parsed view and write into a stack buffer: no
10910+
* raw heap strings to hand back (memory-core ratchet). */
10911+
yyjson_doc *args_doc = args ? yyjson_read(args, strlen(args), 0) : NULL;
10912+
const char *repo_path =
10913+
yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(args_doc), "repo_path"));
10914+
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
10915+
yyjson_mut_val *root = yyjson_mut_obj(doc);
10916+
yyjson_mut_doc_set_root(doc, root);
10917+
yyjson_mut_obj_add_str(doc, root, "status", "error");
10918+
yyjson_mut_obj_add_str(doc, root, "outcome", cbm_proc_outcome_str(CBM_PROC_CLEAN));
10919+
yyjson_mut_obj_add_str(doc, root, "reason", "no_response");
10920+
yyjson_mut_obj_add_strcpy(doc, root, "last_phase", phase);
10921+
yyjson_mut_obj_add_strcpy(doc, root, "worker_log", log);
10922+
char hint[CBM_SZ_1K];
10923+
(void)snprintf(hint, sizeof(hint),
10924+
"Indexing worker exited cleanly (exit 0) without writing a response; the "
10925+
"index was not published. Last phase reached: %s. Inspect the worker log: %s",
10926+
phase, log);
10927+
yyjson_mut_obj_add_strcpy(doc, root, "hint", hint);
10928+
if (repo_path) {
10929+
yyjson_mut_obj_add_strcpy(doc, root, "repo_path", repo_path);
10930+
}
10931+
yyjson_doc_free(args_doc);
10932+
char json[CBM_SZ_16K];
10933+
bool written = yy_mut_normalize_output_text(doc, root, true) &&
10934+
yyjson_mut_write_buf(json, sizeof(json), doc, 0, NULL) > 0;
10935+
yyjson_mut_doc_free(doc);
10936+
/* The fields are bounded (hint 1 KiB, log 4 KiB, phase 128 B), so the
10937+
* plain hint is only the fallback for a pathological repo_path. */
10938+
return cbm_mcp_text_result(written ? json : hint, true);
10939+
}
10940+
1090310941
static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) {
1090410942
invalidate_cached_store(srv);
1090510943

@@ -10911,6 +10949,13 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) {
1091110949
cbm_mcp_supervised_result_disposition_t disposition =
1091210950
cbm_mcp_supervised_result_disposition(rc, &wr);
1091310951

10952+
if (wr.response_missing) {
10953+
/* #1300: a clean exit without a response is a named failure. */
10954+
char *failure = cbm_mcp_index_worker_no_response_failure(args, &wr);
10955+
cbm_index_worker_result_free(&wr);
10956+
invalidate_cached_store(srv);
10957+
return failure;
10958+
}
1091410959
if (disposition == CBM_MCP_SUPERVISED_RESULT_FALLBACK) {
1091510960
cbm_proc_outcome_t outcome = wr.outcome;
1091610961
cbm_index_worker_result_free(&wr);
@@ -10979,6 +11024,13 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) {
1097911024
srv ? &srv->pipeline_cancel_requested : NULL, &wr2);
1098011025
cbm_mcp_supervised_result_disposition_t recovery_disposition =
1098111026
cbm_mcp_supervised_result_disposition(rc2, &wr2);
11027+
if (wr2.response_missing) {
11028+
/* #1300: a recovery run that exits cleanly without a response is
11029+
* reported as such, not as a failed recovery start. */
11030+
resp = cbm_mcp_index_worker_no_response_failure(args, &wr2);
11031+
cbm_index_worker_result_free(&wr2);
11032+
break;
11033+
}
1098211034
if (recovery_disposition == CBM_MCP_SUPERVISED_RESULT_FALLBACK) {
1098311035
/* Keep the original worker failure if recovery setup fails. */
1098411036
cbm_log_error("index.supervisor.recovery_start_failed", "original_outcome",

‎src/mcp/mcp.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,12 @@ char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const ch
244244

245245
/* ── Supervised background index (RSS isolation, #832) ────────── */
246246

247+
/* #1300: the explicit error response for a worker that exited cleanly without
248+
* writing a response (worker_result->response_missing). Names the retained
249+
* worker log and the last phase it reached; caller frees. */
250+
char *cbm_mcp_index_worker_no_response_failure(const char *args,
251+
const cbm_index_worker_result_t *worker_result);
252+
247253
/* One shared policy gate for supervised-index callers. Only an explicitly safe
248254
* terminal can yield a result or enter crash/hang recovery; cancellation,
249255
* failed tree containment, and unavailable worker startup are terminal. The

‎src/pipeline/pipeline_incremental.c‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,25 @@ static const char *itoa_buf(int v) {
102102
return buf[idx];
103103
}
104104

105+
/* #1300: the purge of stale per-file nodes can run for a long time on a large
106+
* persisted graph (reported: hundreds of files against ~268k stored hashes),
107+
* and it logged nothing between incremental.edge_snapshot and
108+
* incremental.purge — a stall there and a clean exit left no trace of where
109+
* the run was. Progress is counts only: a ceiling stride keeps it to at most
110+
* INCR_PURGE_PROGRESS_STEPS lines plus the final one, never one per file. */
111+
enum { INCR_PURGE_PROGRESS_STEPS = 20 };
112+
113+
static void incr_purge_progress(int done, int total, struct timespec start) {
114+
int stride = (total + INCR_PURGE_PROGRESS_STEPS - SKIP_ONE) / INCR_PURGE_PROGRESS_STEPS;
115+
if (stride < SKIP_ONE) {
116+
stride = SKIP_ONE;
117+
}
118+
if (done % stride == 0 || done == total) {
119+
cbm_log_info("incremental.purge.progress", "files_done", itoa_buf(done), "files_total",
120+
itoa_buf(total), "elapsed_ms", itoa_buf((int)elapsed_ms(start)));
121+
}
122+
}
123+
105124
static const char *incr_mode_name(int mode) {
106125
switch (mode) {
107126
case CBM_MODE_FULL:
@@ -2688,12 +2707,17 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
26882707

26892708
/* Step 2: Purge stale nodes */
26902709
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
2710+
int purge_total = ci + deleted_count;
2711+
cbm_log_info("incremental.purge.start", "changed", itoa_buf(ci), "deleted",
2712+
itoa_buf(deleted_count));
26912713
for (int i = 0; i < ci; i++) {
26922714
cbm_gbuf_delete_by_file(existing, changed_files[i].rel_path);
2715+
incr_purge_progress(i + SKIP_ONE, purge_total, t);
26932716
}
26942717
for (int i = 0; i < deleted_count; i++) {
26952718
cbm_gbuf_delete_by_file(existing, deleted[i]);
26962719
free(deleted[i]);
2720+
incr_purge_progress(ci + i + SKIP_ONE, purge_total, t);
26972721
}
26982722
free(deleted);
26992723
cbm_log_info("incremental.purge", "elapsed_ms", itoa_buf((int)elapsed_ms(t)));

‎tests/test_daemon_application.c‎

Lines changed: 83 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1349,6 +1349,8 @@ TEST(daemon_application_prune_clears_logical_watch_for_reregistration) {
13491349

13501350
enum { APP_FAKE_MAX_ATTEMPTS = 16 };
13511351
enum { APP_FAKE_ARGS_CAP = 2048 };
1352+
/* Scripted response slot: a CLEAN exit that wrote no response (#1300). */
1353+
static const char APP_FAKE_NO_RESPONSE[] = "<no response>";
13521354

13531355
typedef struct {
13541356
atomic_int starts;
@@ -1534,7 +1536,16 @@ static cbm_index_worker_poll_t app_fake_worker_poll(void *opaque,
15341536
? worker->context->responses[worker->attempt]
15351537
: "{\"content\":[{\"type\":\"text\",\"text\":\"{"
15361538
"\\\"status\\\":\\\"indexed\\\"}\"}]}";
1537-
worker->result.response = outcome == CBM_PROC_CLEAN ? cbm_strdup(response) : NULL;
1539+
bool silent = response == APP_FAKE_NO_RESPONSE;
1540+
worker->result.response =
1541+
outcome == CBM_PROC_CLEAN && !silent ? cbm_strdup(response) : NULL;
1542+
if (silent) {
1543+
/* What the supervisor reports for a clean exit without a
1544+
* response (#1300). */
1545+
worker->result.response_missing = true;
1546+
(void)snprintf(worker->result.last_phase, sizeof(worker->result.last_phase), "%s",
1547+
"incremental.purge.progress");
1548+
}
15381549
*result_out = &worker->result;
15391550
return CBM_INDEX_WORKER_POLL_TERMINAL;
15401551
}
@@ -3563,6 +3574,76 @@ TEST(daemon_application_disconnect_before_request_callback_is_sticky) {
35633574
PASS();
35643575
}
35653576

3577+
/* #1300: a worker that exits cleanly without a response must reach the
3578+
* client as a named failure: said to be a clean exit with no response, with
3579+
* the last phase it reached and the worker log to inspect — not the generic
3580+
* "ended with clean" line. No recovery re-run for a worker that did not
3581+
* crash. */
3582+
TEST(daemon_application_clean_exit_without_response_is_named_failure_issue1300) {
3583+
app_fake_worker_context_t fake;
3584+
app_fake_worker_context_init(&fake);
3585+
atomic_store(&fake.scripted, true);
3586+
fake.outcomes[0] = CBM_PROC_CLEAN;
3587+
fake.responses[0] = APP_FAKE_NO_RESPONSE;
3588+
cbm_daemon_application_worker_ops_t worker_ops = {
3589+
.context = &fake,
3590+
.start = app_fake_worker_start,
3591+
.poll = app_fake_worker_poll,
3592+
.cancel = app_fake_worker_cancel,
3593+
.log_path = app_fake_worker_log_path,
3594+
.destroy = app_fake_worker_destroy,
3595+
};
3596+
cbm_daemon_application_config_t config = {.worker_ops = &worker_ops};
3597+
cbm_daemon_application_t *application = cbm_daemon_application_new(&config);
3598+
cbm_daemon_runtime_application_callbacks_t callbacks =
3599+
cbm_daemon_application_runtime_callbacks(application);
3600+
cbm_daemon_runtime_application_session_t *session = app_test_open(&callbacks, 1300);
3601+
char root[APP_TEST_PATH_CAP];
3602+
snprintf(root, sizeof(root), "%s/cbm-app-no-response-XXXXXX", cbm_tmpdir());
3603+
bool root_ok = cbm_mkdtemp(root) != NULL;
3604+
uint8_t *context = NULL;
3605+
uint32_t context_length = 0;
3606+
char args[APP_TEST_PATH_CAP + 32];
3607+
snprintf(args, sizeof(args), "{\"repo_path\":\"%s\"}", root);
3608+
uint8_t *tool = NULL;
3609+
uint32_t tool_length = 0;
3610+
bool setup = application && session && root_ok &&
3611+
app_test_context_request(root, root, &context, &context_length) &&
3612+
app_test_tool_request("index_repository", args, &tool, &tool_length);
3613+
uint8_t *response = NULL;
3614+
uint32_t response_length = 0;
3615+
setup = setup && app_test_request(&callbacks, session, context, context_length, &response,
3616+
&response_length) == CBM_DAEMON_RUNTIME_APPLICATION_OK;
3617+
free(response);
3618+
response = NULL;
3619+
response_length = 0;
3620+
cbm_daemon_runtime_application_status_t status =
3621+
setup
3622+
? app_test_request(&callbacks, session, tool, tool_length, &response, &response_length)
3623+
: CBM_DAEMON_RUNTIME_APPLICATION_HANDLER_ERROR;
3624+
if (session) {
3625+
callbacks.session_close(callbacks.context, session);
3626+
}
3627+
bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS);
3628+
cbm_daemon_application_free(application);
3629+
3630+
const char *text = response ? (const char *)response : "";
3631+
ASSERT_TRUE(setup);
3632+
ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_OK);
3633+
ASSERT_TRUE(strstr(text, "without writing a response") != NULL);
3634+
ASSERT_TRUE(strstr(text, "incremental.purge.progress") != NULL);
3635+
ASSERT_TRUE(strstr(text, "/tmp/cbm-fake-worker.log") != NULL);
3636+
ASSERT_TRUE(strstr(text, "indexed") == NULL);
3637+
ASSERT_EQ(atomic_load(&fake.starts), 1);
3638+
ASSERT_TRUE(stopped);
3639+
3640+
free(response);
3641+
free(context);
3642+
free(tool);
3643+
(void)cbm_rmdir(root);
3644+
PASS();
3645+
}
3646+
35663647
TEST(daemon_application_request_cancel_preserves_persistent_watch_and_session) {
35673648
app_watch_race_fixture_t fixture;
35683649
bool fixture_ready = app_watch_race_fixture_init(&fixture, 3021);
@@ -6478,4 +6559,5 @@ SUITE(daemon_application) {
64786559
RUN_TEST(daemon_application_over_budget_response_passes_through_without_recovery);
64796560
RUN_TEST(daemon_application_free_reports_retained_live_ownership);
64806561
RUN_TEST(daemon_application_rejects_clean_exit_when_process_tree_is_not_contained);
6562+
RUN_TEST(daemon_application_clean_exit_without_response_is_named_failure_issue1300);
64816563
}

0 commit comments

Comments
 (0)