Merge pull request #1920 from OhOkThisIsFine/claude/focused-herschel-ee8e1c

fix(daemon): contain zombie generations from abandoned requests, name mute endpoint holders

Author: DeusData

src/daemon/bootstrap.c

@@ -359,11 +359,20 @@ static cbm_daemon_bootstrap_status_t bootstrap_finish_probe(
 
 static cbm_daemon_bootstrap_probe_status_t bootstrap_probe(
     const cbm_daemon_bootstrap_config_t *config, const cbm_daemon_bootstrap_ops_t *ops,
-    cbm_daemon_runtime_client_t **client_out, cbm_daemon_runtime_connect_result_t *connect_result) {
+    cbm_daemon_runtime_client_t **client_out, cbm_daemon_runtime_connect_result_t *connect_result,
+    uint64_t *muted_holder_pid_io) {
     memset(connect_result, 0, sizeof(*connect_result));
     *client_out = NULL;
-    return ops->probe(ops->context, config->endpoint, config->identity, config->connect_timeout_ms,
-                      client_out, connect_result);
+    cbm_daemon_bootstrap_probe_status_t status =
+        ops->probe(ops->context, config->endpoint, config->identity, config->connect_timeout_ms,
+                   client_out, connect_result);
+    /* Sticky across probes: a later fast-path probe never attempts a connect
+     * and reports no holder, but a mute holder seen once must survive into
+     * the final timeout diagnostic. */
+    if (muted_holder_pid_io && connect_result->muted_endpoint_holder_pid != 0) {
+        *muted_holder_pid_io = connect_result->muted_endpoint_holder_pid;
+    }
+    return status;
 }
 
 static bool bootstrap_probe_is_finishable(cbm_daemon_bootstrap_probe_status_t probe) {
@@ -432,8 +441,9 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
 
     cbm_daemon_runtime_client_t *client = NULL;
     cbm_daemon_runtime_connect_result_t connect_result;
+    uint64_t muted_holder_pid = 0;
     cbm_daemon_bootstrap_probe_status_t probe =
-        bootstrap_probe(config, ops, &client, &connect_result);
+        bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
     if (bootstrap_probe_is_finishable(probe)) {
         cbm_daemon_bootstrap_status_t status =
             bootstrap_finish_probe(probe, client, &connect_result, ops, result_out);
@@ -461,7 +471,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
              * replacements from being launched. */
             generation_observed = true;
             bootstrap_pause(deadline);
-            probe = bootstrap_probe(config, ops, &client, &connect_result);
+            probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
             continue;
         }
 
@@ -473,7 +483,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
         }
         if (lock_status == 0) {
             bootstrap_pause(deadline);
-            probe = bootstrap_probe(config, ops, &client, &connect_result);
+            probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
             continue;
         }
         if (lock_status != 1 || !startup_lock) {
@@ -482,7 +492,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
         }
 
         lock_acquired = true;
-        probe = bootstrap_probe(config, ops, &client, &connect_result);
+        probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
         if (probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED ||
             probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL) {
             generation_observed = true;
@@ -516,7 +526,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
          * bootstrap against a daemon that is trying to cleanly stand down. */
         do {
             bootstrap_pause(deadline);
-            probe = bootstrap_probe(config, ops, &client, &connect_result);
+            probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
             if (probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED ||
                 probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL) {
                 generation_observed = true;
@@ -545,7 +555,16 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
     }
 
     result_out->status = CBM_DAEMON_BOOTSTRAP_FAILED;
-    if (generation_observed) {
+    if (muted_holder_pid != 0) {
+        /* The one diagnostic the 2026-08-29 zombie recovery had to assemble by
+         * hand from process, pipe, and log correlation: name the pid that
+         * holds the endpoint without answering, and say what to do with it. */
+        (void)snprintf(result_out->message, sizeof(result_out->message),
+                       "CBM daemon endpoint is held by pid %llu but that process answered no "
+                       "rendezvous within %u ms; the daemon runtime is likely dead — stop that "
+                       "process, then retry",
+                       (unsigned long long)muted_holder_pid, config->startup_timeout_ms);
+    } else if (generation_observed) {
         (void)snprintf(result_out->message, sizeof(result_out->message),
                        "CBM daemon is active or starting but could not accept this client "
                        "within %u ms",
@@ -578,6 +597,16 @@ cbm_daemon_bootstrap_probe_status_t cbm_daemon_bootstrap_classify_failed_connect
          * existing generation answered. Never reinterpret them as absence. */
         return CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED;
     }
+    if (connect_result->status == CBM_DAEMON_RUNTIME_CONNECT_ERROR &&
+        connect_result->muted_endpoint_holder_pid != 0) {
+        /* The transport connected to a live process that answered nothing.
+         * That endpoint is OWNED, whatever the advisory locks read right now
+         * (a wedged generation can hold pipes while its lock files churn).
+         * Classifying this as absence made the 2026-08-29 zombie invisible:
+         * the starter spawned doomed competitors for 30 s and then reported a
+         * bare timeout with no holder named. */
+        return CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED;
+    }
     if (lifetime_status == 1) {
         return CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED;
     }

src/daemon/frontend.c

@@ -40,6 +40,19 @@ enum {
      * cancelled and the frontend exits cleanly. */
     FRONTEND_EOF_DRAIN_MS = 15000,
     FRONTEND_MAINTENANCE_POLL_MS = 10,
+    /* Cadence for the quiet no-maintenance steady state. Every presence probe
+     * try-acquires the maintenance marker lock, and on Windows that
+     * revalidates the whole ancestor directory chain (a CreateFileW +
+     * identity check per path component). At the old 10 ms cadence this
+     * burned ~20% of a core for the life of every MCP session — 7762
+     * CPU-seconds on one 9.5 h session in the 2026-08-29 incident. The
+     * budget: an activation drains participants within
+     * CLI_ACTIVATION_DRAIN_TIMEOUT_MS (15 s) and the monitor's own exit grace
+     * is FRONTEND_MAINTENANCE_GRACE_MS (3 s), so a 250 ms detection latency
+     * spends under 2% of that window while cutting the probe cost 25-fold.
+     * The 10 ms cadence still paces the monitor's post-detection grace loop,
+     * which does no lock probing. */
+    FRONTEND_MAINTENANCE_IDLE_POLL_MS = 250,
     /* The owner thread may be draining a supervised process tree. Preserve the
      * supervisor's complete graceful + forced-settle window before the monitor
      * fail-stops the process, plus scheduling/teardown margin. */
@@ -162,7 +175,7 @@ static void *frontend_maintenance_monitor_worker(void *opaque) {
         cbm_version_cohort_maintenance_presence_t presence =
             frontend_observe_maintenance(monitor->manager, true);
         if (presence == CBM_VERSION_COHORT_MAINTENANCE_ABSENT) {
-            cbm_usleep(FRONTEND_MAINTENANCE_POLL_MS * 1000U);
+            cbm_usleep(FRONTEND_MAINTENANCE_IDLE_POLL_MS * 1000U);
             continue;
         }
 

src/daemon/runtime.c

@@ -36,6 +36,19 @@ static atomic_bool runtime_force_peer_image_mismatch_seam;
 void cbm_daemon_runtime_force_peer_image_mismatch_for_testing(bool force) {
     atomic_store(&runtime_force_peer_image_mismatch_seam, force);
 }
+/* The abandoned-request containment path ends in process termination, which an
+ * in-process harness cannot observe. The timeout override makes the ceiling
+ * reachable in test time; the hook replaces termination with a recordable
+ * callback. Both stay inert (zero/NULL) outside tests. */
+static _Atomic uint32_t runtime_abandoned_request_join_timeout_seam;
+void cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(uint32_t timeout_ms) {
+    atomic_store(&runtime_abandoned_request_join_timeout_seam, timeout_ms);
+}
+static _Atomic(cbm_daemon_runtime_containment_hook_t) runtime_containment_hook_seam;
+void cbm_daemon_runtime_set_containment_hook_for_testing(
+    cbm_daemon_runtime_containment_hook_t hook) {
+    atomic_store(&runtime_containment_hook_seam, hook);
+}
 #endif
 
 #ifdef _WIN32
@@ -72,6 +85,16 @@ enum {
      * a local cooperative peer is already blocked in read and consumes the
      * rejection within milliseconds. */
     RUNTIME_REJECT_DRAIN_TIMEOUT_MS = 250,
+    /* Ceiling on how long a disconnecting worker waits for its in-flight
+     * application request to observe cancellation. session_cancel has already
+     * run by the time this wait starts, so a compliant handler returns within
+     * milliseconds; the wait is sized for a handler that only polls its cancel
+     * flag between long pipeline stages. A handler that ignores cancellation
+     * past this ceiling turned the daemon into a permanent zombie once
+     * (2026-08-29): the unbounded join blocked the disconnect path, admission
+     * wedged behind it, and the dead generation held the endpoint pipes and
+     * the UI port for nine hours with nothing logged. */
+    RUNTIME_ABANDONED_REQUEST_JOIN_TIMEOUT_MS = 30000,
     RUNTIME_PATH_CAP = 4096,
 
     RENDEZVOUS_REQUEST_ABI_OFFSET = 0,
@@ -1313,13 +1336,62 @@ static void *runtime_application_worker(void *opaque) {
     return NULL;
 }
 
+static _Noreturn void runtime_cleanup_fail_stop(const char *component);
+
+static void runtime_contain_unresponsive_application(cbm_daemon_runtime_worker_t *worker) {
+    char peer_pid[32];
+    char token[32];
+    (void)snprintf(peer_pid, sizeof(peer_pid), "%llu", (unsigned long long)worker->peer_process_id);
+    (void)snprintf(token, sizeof(token), "%llu",
+                   (unsigned long long)worker->application_request_token);
+    cbm_log_error("daemon.application_request_unresponsive", "peer_pid", peer_pid, "request_token",
+                  token);
+#ifdef CBM_ENABLE_TEST_SEAMS
+    cbm_daemon_runtime_containment_hook_t hook = atomic_load(&runtime_containment_hook_seam);
+    if (hook) {
+        hook("application_request_join");
+        return;
+    }
+#endif
+    runtime_cleanup_fail_stop("application_request_join");
+}
+
 static bool runtime_worker_reap_application(cbm_daemon_runtime_worker_t *worker, bool wait) {
     if (!worker->application_thread_started) {
         return true;
     }
     if (!wait && !atomic_load_explicit(&worker->application_thread_done, memory_order_acquire)) {
         return false;
     }
+    if (!atomic_load_explicit(&worker->application_thread_done, memory_order_acquire)) {
+        /* The peer is gone and session_cancel already ran, so the handler owes
+         * a prompt return; only a handler that ignores cancellation reaches
+         * the ceiling. An unbounded join here once turned one wedged request
+         * into a whole-daemon zombie (2026-08-29): the disconnect path
+         * blocked, the worker slot never released, and the dead generation
+         * held the endpoint pipes and UI port for nine hours with nothing
+         * logged while every new client timed out bare. Containment
+         * terminates the process instead — the kernel releases every native
+         * claim and the next client starts a fresh generation. */
+        uint32_t timeout_ms = RUNTIME_ABANDONED_REQUEST_JOIN_TIMEOUT_MS;
+#ifdef CBM_ENABLE_TEST_SEAMS
+        uint32_t seam_timeout = atomic_load(&runtime_abandoned_request_join_timeout_seam);
+        if (seam_timeout != 0) {
+            timeout_ms = seam_timeout;
+        }
+#endif
+        uint64_t deadline = runtime_deadline_after(timeout_ms);
+        while (!atomic_load_explicit(&worker->application_thread_done, memory_order_acquire)) {
+            if (cbm_now_ms() >= deadline) {
+                /* Terminal in production. A test containment hook returns, and
+                 * the join then waits for the harness to release the handler
+                 * so teardown stays leak-free under the sanitizers. */
+                runtime_contain_unresponsive_application(worker);
+                deadline = UINT64_MAX;
+            }
+            runtime_wait_tick(deadline);
+        }
+    }
     if (cbm_thread_join(&worker->application_thread) != 0) {
         return false;
     }
@@ -2569,8 +2641,11 @@ static bool runtime_control_request_send(const cbm_daemon_ipc_endpoint_t *endpoi
                                          const cbm_daemon_build_identity_t *identity,
                                          cbm_daemon_runtime_operation_t operation,
                                          uint32_t timeout_ms, uint32_t response_size,
-                                         uint8_t **payload_out) {
+                                         uint8_t **payload_out, uint64_t *muted_holder_pid_out) {
     *payload_out = NULL;
+    if (muted_holder_pid_out) {
+        *muted_holder_pid_out = 0;
+    }
     if (!endpoint || !identity || !identity->build_fingerprint ||
         timeout_ms == CBM_DAEMON_IPC_WAIT_FOREVER) {
         return false;
@@ -2590,6 +2665,15 @@ static bool runtime_control_request_send(const cbm_daemon_ipc_endpoint_t *endpoi
     int received = sent ? cbm_daemon_ipc_receive_frame_bounded(connection, timeout_ms,
                                                                response_size, &frame, &payload)
                         : 0;
+    if (muted_holder_pid_out && sent &&
+        (received != 1 || (frame.type == CBM_DAEMON_FRAME_RESPONSE && frame.flags != operation))) {
+        /* Connected but not served: either total silence (dead runtime), or a
+         * wrong-operation reject frame (a wedged generation whose accept path
+         * still answers inline while every worker slot is stuck). Both are
+         * the zombie class — surface the holder's kernel-reported pid so
+         * `daemon status` can name it instead of reporting "not running". */
+        *muted_holder_pid_out = cbm_daemon_ipc_connection_peer_pid(connection);
+    }
     bool valid = received == 1 && frame.type == CBM_DAEMON_FRAME_RESPONSE &&
                  frame.flags == operation && frame.length == response_size && payload &&
                  payload[0] == 1U;
@@ -2617,7 +2701,8 @@ bool cbm_daemon_runtime_request_status(const cbm_daemon_ipc_endpoint_t *endpoint
     memset(status_out, 0, sizeof(*status_out));
     uint8_t *payload = NULL;
     if (!runtime_control_request_send(endpoint, identity, CBM_DAEMON_RUNTIME_OP_STATUS, timeout_ms,
-                                      CBM_DAEMON_STATUS_RESPONSE_SIZE, &payload)) {
+                                      CBM_DAEMON_STATUS_RESPONSE_SIZE, &payload,
+                                      &status_out->muted_endpoint_holder_pid)) {
         return false;
     }
     status_out->permanent = (payload[1] & 0x01U) != 0U;
@@ -2648,7 +2733,7 @@ bool cbm_daemon_runtime_request_stop(const cbm_daemon_ipc_endpoint_t *endpoint,
     memset(result_out, 0, sizeof(*result_out));
     uint8_t *payload = NULL;
     if (!runtime_control_request_send(endpoint, identity, CBM_DAEMON_RUNTIME_OP_STOP, timeout_ms,
-                                      CBM_DAEMON_STOP_RESPONSE_SIZE, &payload)) {
+                                      CBM_DAEMON_STOP_RESPONSE_SIZE, &payload, NULL)) {
         return false;
     }
     result_out->accepted = (payload[1] & 0x01U) != 0U;
@@ -2691,6 +2776,13 @@ cbm_daemon_runtime_client_t *cbm_daemon_runtime_client_connect(
                  frame.length == CBM_DAEMON_RENDEZVOUS_RESPONSE_SIZE &&
                  runtime_hello_response_decode(payload, result_out);
     free(payload);
+    if (received != 1) {
+        /* The kernel completed the pipe/socket connect, so a server process
+         * exists, yet the HELLO went unanswered. Name that holder: a dead
+         * runtime behind a live endpoint is otherwise indistinguishable from
+         * absence, and the 2026-08-29 zombie hid behind exactly that gap. */
+        result_out->muted_endpoint_holder_pid = cbm_daemon_ipc_connection_peer_pid(connection);
+    }
     if (!valid || result_out->status != CBM_DAEMON_RUNTIME_CONNECT_ACCEPTED) {
         cbm_daemon_ipc_connection_close(connection);
         return NULL;

src/daemon/runtime.h

@@ -214,6 +214,12 @@ typedef struct {
     cbm_daemon_client_id_t client_id;
     /* Kernel-authenticated PID of this client as observed by the daemon. */
     uint64_t authenticated_process_id;
+    /* Nonzero only when the transport connect succeeded but no valid response
+     * arrived in time: the kernel-reported PID of the process that holds the
+     * endpoint. A held-but-mute endpoint is a live process whose runtime is
+     * wedged (2026-08-29 zombie incident) — callers must report this PID
+     * rather than fold the failure into plain absence. */
+    uint64_t muted_endpoint_holder_pid;
     cbm_daemon_conflict_t conflict;
     char message[CBM_DAEMON_CONFLICT_MESSAGE_SIZE];
 } cbm_daemon_runtime_connect_result_t;
@@ -265,6 +271,11 @@ typedef struct {
     uint32_t client_pids[CBM_DAEMON_CONTROL_CLIENT_CAP];
     char build_fingerprint[CBM_DAEMON_BUILD_FINGERPRINT_SIZE];
     char semantic_version[CBM_DAEMON_SEMVER_SIZE];
+    /* Meaningful even when the status request itself FAILED: nonzero when the
+     * endpoint accepted the transport connect but answered nothing — the
+     * kernel-reported PID of the mute holder. `daemon status` uses this to
+     * name a zombie generation instead of reporting bare "not running". */
+    uint64_t muted_endpoint_holder_pid;
 } cbm_daemon_runtime_status_t;
 
 typedef struct {
@@ -421,6 +432,15 @@ void cbm_daemon_runtime_force_peer_image_unverified_for_testing(bool force);
  * i.e. the tamper case that must still be rejected after unverifiable images
  * became admissible. */
 void cbm_daemon_runtime_force_peer_image_mismatch_for_testing(bool force);
+/* Abandoned-request containment seams (2026-08-29 zombie incident). The
+ * timeout override shrinks the join ceiling to test scale; zero restores the
+ * production constant. The hook replaces the terminal containment stop with a
+ * recordable callback so an in-process harness survives the trigger; NULL
+ * restores process termination. */
+typedef void (*cbm_daemon_runtime_containment_hook_t)(const char *component);
+void cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(uint32_t timeout_ms);
+void cbm_daemon_runtime_set_containment_hook_for_testing(
+    cbm_daemon_runtime_containment_hook_t hook);
 #endif
 
 #endif /* CBM_DAEMON_RUNTIME_H */

src/daemon/version_cohort.c

@@ -463,7 +463,11 @@ cbm_version_cohort_status_t cbm_version_cohort_acquire(cbm_version_cohort_manage
          * which a mismatched client can join. Name it: "was the lifetime lock
          * held?" is exactly what separates a local run (conflict raised) from
          * a CI run (client admitted), and it was not observable in any log. */
-        cbm_log_info("version_cohort.claimed_unheld", "build",
+        /* Key wording matters here: the old name "claimed_unheld" read as a
+         * stale-claim anomaly and derailed the 2026-08-29 zombie diagnosis,
+         * when the record actually marks the HEALTHY fresh-claim path that
+         * every normal start logs. */
+        cbm_log_info("version_cohort.claimed_fresh", "prior_holder", "none", "build",
                      identity->build_fingerprint ? identity->build_fingerprint : "<null>");
         status = version_cohort_claim_new(lease, identity, deadline_ms);
     } else if (status == CBM_VERSION_COHORT_BUSY) {