fix(daemon): re-issue the interrupted request against a replacement daemon

When the shared daemon dies under a live MCP frontend (#2796), the worker's
daemon exchange fails and the current failure path calls _Exit(EXIT_FAILURE)
— "a failed daemon transport cannot wake a thread blocked in stdio
portably". The agent then sees server EOF and every pooled connection to the
same daemon breaks at once. On Windows this happens whenever a client reaps
its spawned endpoint tree: DETACHED_PROCESS detaches the console, not the
process tree, so the shared daemon dies with the endpoint that spawned it.

Keep the reader untouched and recover on the worker instead: when
cbm_daemon_application_client_mcp_tagged fails and the caller supplied a
replacement supplier (new daemon_lost callback on mcp_run), swap in the
replacement client, re-mint the request token (tokens are scoped to the
client that reserved them), and re-issue the very request whose exchange was
interrupted. Bounded by FRONTEND_DAEMON_RECONNECT_MAX; without a callback or
once the budget is spent the old immediate-exit path stays. A cancellation
that arrives mid-reconnect targets the old token and is dropped — the
re-issued request completes instead.

main() supplies the callback: it owns the dead client (closes it), re-runs
the connect-or-spawn leg plus session context, and returns the replacement.
UI flags are not re-applied; the daemon persists its UI configuration.

Verified on WSL/gcc 13.3 with a real daemon: two stdio endpoints sharing one
daemon, kill -9 the daemon, the next request on the survivor is served by a
re-spawned daemon (stderr: one daemon_lost_reconnect warning); pre-fix the
same scenario exits 1 and the request is lost. scripts/test.sh --suites
daemon_frontend stays green.

Fixes #2796

Signed-off-by: Minervaowl7 <127482476+Minervaowl7@users.noreply.github.com>

Author: Minervaowl7

src/daemon/frontend.c

@@ -39,6 +39,10 @@ enum {
      * receiving every response while a genuinely stuck request still gets
      * cancelled and the frontend exits cleanly. */
     FRONTEND_EOF_DRAIN_MS = 15000,
+    /* #2796: replacement-daemon attempts per frontend run once the transport
+     * fails mid-session. Bounded so a daemon that cannot come back still
+     * terminates the process exactly like the pre-reconnect behavior. */
+    FRONTEND_DAEMON_RECONNECT_MAX = 3,
     FRONTEND_MAINTENANCE_POLL_MS = 10,
     /* Cadence for the quiet no-maintenance steady state. Every presence probe
      * try-acquires the maintenance marker lock, and on Windows that
@@ -90,6 +94,12 @@ typedef struct {
     const char *active_id_str;
     cbm_daemon_runtime_application_token_t active_request_token;
     bool failed;
+    /* #2796: replacement supplier for a dead daemon transport, plus the retry
+     * budget already spent in this run. daemon_lost==NULL keeps the legacy
+     * fatal behavior. */
+    cbm_daemon_frontend_daemon_lost_fn daemon_lost;
+    void *daemon_lost_context;
+    unsigned int daemon_reconnects;
     /* Monotonic count of fully processed queue items (responses written or
      * cancellations acknowledged). The EOF drain below watches it to tell a
      * batch that is still making progress from a genuinely stuck request. */
@@ -502,6 +512,65 @@ static frontend_cancellation_route_t frontend_route_cancellation(
     return route;
 }
 
+/* #2796: swap in a replacement daemon client after a transport failure. The
+ * request token minted for the interrupted exchange was scoped to the dead
+ * client, so the caller must re-reserve one before re-issuing the request.
+ * The callback runs unmutexed (a cold daemon spawn takes seconds); the swap
+ * itself refuses once teardown has begun so no replacement outlives the run.
+ * Every other state->client reader either holds the mutex or is this worker
+ * thread between exchanges. */
+static bool frontend_daemon_reconnect(frontend_state_t *state) {
+    if (!state->daemon_lost || state->daemon_reconnects >= FRONTEND_DAEMON_RECONNECT_MAX ||
+        state->stopping) {
+        return false;
+    }
+    cbm_daemon_runtime_client_t *dead = state->client;
+    state->daemon_reconnects++;
+    cbm_daemon_runtime_client_t *replacement = state->daemon_lost(state->daemon_lost_context, dead);
+    if (!replacement) {
+        return false;
+    }
+    cbm_mutex_lock(&state->mutex);
+    if (state->stopping || state->client != dead) {
+        cbm_mutex_unlock(&state->mutex);
+        (void)cbm_daemon_runtime_client_close(replacement, FRONTEND_CLOSE_TIMEOUT_MS);
+        return false;
+    }
+    state->client = replacement;
+    cbm_mutex_unlock(&state->mutex);
+    return true;
+}
+
+/* #2796: one request exchange with replacement-daemon retries. Returns the
+ * final status; response and response_length are set only on OK. */
+static cbm_daemon_runtime_application_status_t frontend_exchange_with_reconnect(
+    frontend_state_t *state, frontend_item_t *item, uint8_t **response, uint32_t *response_length) {
+    cbm_daemon_runtime_application_status_t status;
+    for (;;) {
+        status = cbm_daemon_application_client_mcp_tagged(state->client, item->request_token,
+                                                          item->message, response, response_length,
+                                                          FRONTEND_REQUEST_TIMEOUT_MS);
+        if (status == CBM_DAEMON_RUNTIME_APPLICATION_OK) {
+            return status;
+        }
+        /* A dead daemon transport is not this session's death: swap in a
+         * replacement client, re-mint the request token (tokens are scoped to
+         * the client that reserved them), and re-issue the very request whose
+         * exchange was interrupted. */
+        if (!frontend_daemon_reconnect(state) ||
+            !cbm_daemon_runtime_client_application_token_reserve(state->client,
+                                                                 &item->request_token)) {
+            return status;
+        }
+        cbm_mutex_lock(&state->mutex);
+        state->active_request_token = item->request_token;
+        cbm_mutex_unlock(&state->mutex);
+        free(*response);
+        *response = NULL;
+        *response_length = 0;
+    }
+}
+
 static void *frontend_worker(void *opaque) {
     frontend_state_t *state = opaque;
     for (;;) {
@@ -523,9 +592,7 @@ static void *frontend_worker(void *opaque) {
             failed = !frontend_write_cancelled_response(state->out, &item);
         } else {
             cbm_daemon_runtime_application_status_t status =
-                cbm_daemon_application_client_mcp_tagged(state->client, item.request_token,
-                                                         item.message, &response, &response_length,
-                                                         FRONTEND_REQUEST_TIMEOUT_MS);
+                frontend_exchange_with_reconnect(state, &item, &response, &response_length);
             if (status == CBM_DAEMON_RUNTIME_APPLICATION_CANCELLED) {
                 failed = !frontend_write_cancelled_response(state->out, &item);
             } else {
@@ -544,12 +611,14 @@ static void *frontend_worker(void *opaque) {
         atomic_fetch_add_explicit(&state->completed_items, 1, memory_order_release);
         if (failed) {
             if (!expected_stop) {
-                /* A failed daemon transport cannot wake a thread blocked in
-                 * stdio portably. This frontend owns no state: immediate
-                 * process exit closes the kernel IPC handle, which cancels
-                 * daemon session ownership without a detached reader or an
-                 * unsafe cross-thread fclose. Logging or flushing here could
-                 * itself block on agent-owned stdout/stderr. */
+                /* #2796: reached only without a daemon_lost callback or after
+                 * its retry budget is spent. A failed daemon transport cannot
+                 * wake a thread blocked in stdio portably. This frontend owns
+                 * no state: immediate process exit closes the kernel IPC
+                 * handle, which cancels daemon session ownership without a
+                 * detached reader or an unsafe cross-thread fclose. Logging or
+                 * flushing here could itself block on agent-owned
+                 * stdout/stderr. */
                 _Exit(EXIT_FAILURE);
             }
             break;
@@ -657,14 +726,18 @@ static bool frontend_cancel_for_maintenance(void *opaque) {
 }
 
 int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client,
-                                cbm_version_cohort_manager_t *cohort_manager, FILE *in, FILE *out) {
+                                cbm_version_cohort_manager_t *cohort_manager, FILE *in, FILE *out,
+                                cbm_daemon_frontend_daemon_lost_fn daemon_lost,
+                                void *daemon_lost_context) {
     if (!client || !cohort_manager || !in || !out) {
         return -1;
     }
     frontend_state_t state = {
         .client = client,
         .cohort_manager = cohort_manager,
         .out = out,
+        .daemon_lost = daemon_lost,
+        .daemon_lost_context = daemon_lost_context,
     };
     cbm_mutex_init(&state.mutex);
     atomic_init(&state.completed_items, 0);

src/daemon/frontend.h

@@ -17,6 +17,14 @@ typedef struct cbm_daemon_maintenance_monitor cbm_daemon_maintenance_monitor_t;
  * outlive the bounded grace period. */
 typedef bool (*cbm_daemon_maintenance_cancel_fn)(void *context);
 
+/* #2796: invoked on the frontend worker thread when the daemon transport
+ * fails while the MCP client's session is still live. The callback owns the
+ * dead client (it must close it), establishes a replacement (connect-or-spawn
+ * plus session context), and returns it — NULL leaves the transport failure
+ * fatal. It must not touch the frontend's stdio streams. */
+typedef cbm_daemon_runtime_client_t *(*cbm_daemon_frontend_daemon_lost_fn)(
+    void *context, cbm_daemon_runtime_client_t *dead_client);
+
 /* Parse one JSON-RPC message and recognize only the exact cancellation
  * notification method. String contents, nested fields, method prefixes, and
  * requests carrying an id are not cancellation notifications. */
@@ -66,9 +74,14 @@ uint64_t cbm_daemon_frontend_test_worker_idle_cycles(void);
  * either thread is blocked in stdio, requests cooperative cancellation for the
  * exact active request, and then bounds process exit. Kernel IPC close cancels
  * only this session's daemon work. EOF/parse failure closes the authenticated
- * session. An unexpected daemon transport failure likewise terminates the
- * process so an agent waiting with stdin still open observes server EOF. */
+ * session. An unexpected daemon transport failure is retried against a
+ * replacement daemon through daemon_lost (bounded, #2796); without the
+ * callback — or once the retry budget is spent — the worker terminates the
+ * process immediately so an agent waiting with stdin still open observes
+ * server EOF. */
 int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client,
-                                cbm_version_cohort_manager_t *cohort_manager, FILE *in, FILE *out);
+                                cbm_version_cohort_manager_t *cohort_manager, FILE *in, FILE *out,
+                                cbm_daemon_frontend_daemon_lost_fn daemon_lost,
+                                void *daemon_lost_context);
 
 #endif /* CBM_DAEMON_FRONTEND_H */

src/main.c

@@ -2785,6 +2785,53 @@ static int main_run_daemon_ctl(int argc, char **argv, const cbm_daemon_ipc_endpo
     return ui_result;
 }
 
+/* #2796: supplier for the MCP frontend's daemon_lost callback. Owns the dead
+ * client (closes it), re-runs the whole connect-or-spawn leg with the session
+ * context, and hands the frontend the replacement. Runs on the frontend worker
+ * thread; reports to stderr only and never touches the agent's stdio. UI flags
+ * are not re-applied: the daemon persists its UI configuration and a re-spawned
+ * daemon reads it back. */
+typedef struct {
+    cbm_daemon_process_role_t role;
+    const cbm_daemon_build_identity_t *identity;
+    const char *executable_path;
+    cbm_mcp_tool_profile_t tool_profile;
+} main_mcp_daemon_lost_context_t;
+
+static cbm_daemon_runtime_client_t *main_mcp_daemon_lost(void *context,
+                                                         cbm_daemon_runtime_client_t *dead_client) {
+    main_mcp_daemon_lost_context_t *lost = context;
+    (void)cbm_daemon_runtime_client_close(dead_client, MAIN_CLOSE_TIMEOUT_MS);
+    (void)fprintf(stderr, "level=warn msg=mcp.frontend.daemon_lost_reconnect\n");
+    cbm_daemon_ipc_endpoint_t *endpoint = main_daemon_endpoint_new();
+    if (!endpoint) {
+        return NULL;
+    }
+    cbm_daemon_bootstrap_config_t config = {
+        .role = lost->role,
+        .endpoint = endpoint,
+        .identity = lost->identity,
+        .executable_path = lost->executable_path,
+        .connect_timeout_ms = MAIN_CONNECT_TIMEOUT_MS,
+        .startup_timeout_ms = main_startup_timeout_ms(),
+    };
+    cbm_daemon_bootstrap_result_t result;
+    cbm_daemon_bootstrap_status_t status =
+        main_client_bootstrap_with_upgrade(&config, &result, true);
+    cbm_daemon_ipc_endpoint_free(endpoint);
+    if (status != CBM_DAEMON_BOOTSTRAP_CONNECTED || !result.client) {
+        main_report_client_bootstrap_failure(lost->role, &result);
+        return NULL;
+    }
+    if (!main_set_client_context(result.client, NULL, lost->tool_profile, NULL, NULL,
+                                 config.startup_timeout_ms)) {
+        (void)fprintf(stderr, "codebase-memory-mcp: daemon session context was rejected\n");
+        (void)cbm_daemon_runtime_client_close(result.client, MAIN_CLOSE_TIMEOUT_MS);
+        return NULL;
+    }
+    return result.client;
+}
+
 int main(int argc, char **argv) {
     /* Must remain the first statement: see allocator binding contract above. */
     cbm_alloc_init();
@@ -3433,7 +3480,14 @@ int main(int argc, char **argv) {
     }
 
     setup_signal_handlers();
-    int result = cbm_daemon_frontend_mcp_run(g_daemon_client, client_cohort_manager, stdin, stdout);
+    main_mcp_daemon_lost_context_t daemon_lost_context = {
+        .role = role,
+        .identity = &identity,
+        .executable_path = executable_path,
+        .tool_profile = tool_profile,
+    };
+    int result = cbm_daemon_frontend_mcp_run(g_daemon_client, client_cohort_manager, stdin, stdout,
+                                             main_mcp_daemon_lost, &daemon_lost_context);
     g_daemon_client = NULL; /* frontend consumed the handle */
     bool client_cohort_cleanup =
         main_version_cohort_close(&client_cohort_lease, &client_cohort_manager);

tests/test_daemon_frontend.c

@@ -213,7 +213,7 @@ static int frontend_test_close_fd(int fd) {
 static void *frontend_idle_run(void *opaque) {
     frontend_idle_run_t *run = opaque;
     run->result =
-        cbm_daemon_frontend_mcp_run(run->client, run->manager, run->input, run->output);
+        cbm_daemon_frontend_mcp_run(run->client, run->manager, run->input, run->output, NULL, NULL);
     return NULL;
 }
 #endif
@@ -579,7 +579,7 @@ static int frontend_eof_child_run(const char *parent, bool overflow) {
 
     cbm_daemon_runtime_client_t *frontend_client = fixture.client;
     fixture.client = NULL; /* cbm_daemon_frontend_mcp_run consumes it. */
-    int result = cbm_daemon_frontend_mcp_run(frontend_client, fixture.manager, input, output);
+    int result = cbm_daemon_frontend_mcp_run(frontend_client, fixture.manager, input, output, NULL, NULL);
     bool joined = cbm_thread_join(&writer_thread) == 0;
     bool writer_ok = atomic_load_explicit(&writer.finished, memory_order_acquire) &&
                      atomic_load_explicit(&writer.succeeded, memory_order_acquire);
@@ -920,7 +920,7 @@ static int frontend_backpressure_frontend_run(const char *parent, int input_fd,
         return 81;
     }
 
-    int result = cbm_daemon_frontend_mcp_run(client, manager, input, output);
+    int result = cbm_daemon_frontend_mcp_run(client, manager, input, output, NULL, NULL);
     bool joined = cbm_thread_join(&writer_thread) == 0;
     bool writer_ok = atomic_load_explicit(&writer.finished, memory_order_acquire) &&
                      atomic_load_explicit(&writer.succeeded, memory_order_acquire);
@@ -1419,7 +1419,7 @@ TEST(daemon_frontend_maintenance_exits_while_stdio_reader_is_blocked) {
             _exit(70);
         }
         int result = cbm_daemon_frontend_mcp_run((cbm_daemon_runtime_client_t *)(uintptr_t)1,
-                                                 manager, input, output);
+                                                 manager, input, output, NULL, NULL);
         (void)result;
         _exit(71);
     }