Skip to content

Commit c725453

Browse files
authored
Merge pull request DeusData#2179 from DeusData/fix/daemon-ephemeral-retire-cohort
fix(daemon): keep an ephemeral generation for a mid-HELLO cold-storm racer
2 parents 055fbb7 + 6091db7 commit c725453

6 files changed

Lines changed: 308 additions & 3 deletions

File tree

‎src/daemon/daemon.c‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,11 @@ struct cbm_daemon_coordinator {
4747
size_t client_count;
4848
/* See cbm_daemon_coordinator_set_permanent. */
4949
bool permanent;
50+
/* See cbm_daemon_coordinator_set_linger. Distinct from `permanent`: a
51+
* transient, runtime-driven hold that keeps an ephemeral coordinator
52+
* admitting while cohort participants are still mid-bootstrap (cold-storm
53+
* race, 2026-09). Clearing it while idle retires the coordinator at once. */
54+
bool linger;
5055
size_t job_count;
5156
size_t watch_count;
5257
size_t callback_count;
@@ -291,7 +296,7 @@ static void release_client_locked(cbm_daemon_coordinator_t *coordinator,
291296
release_client_resources_locked(coordinator, client->id, batch);
292297
free(client);
293298
coordinator->client_count--;
294-
if (coordinator->client_count == 0 && !coordinator->permanent) {
299+
if (coordinator->client_count == 0 && !coordinator->permanent && !coordinator->linger) {
295300
coordinator->state = CBM_DAEMON_COORDINATOR_STOPPING;
296301
}
297302
}
@@ -329,6 +334,23 @@ void cbm_daemon_coordinator_set_permanent(cbm_daemon_coordinator_t *coordinator,
329334
cbm_mutex_unlock(&coordinator->mutex);
330335
}
331336

337+
void cbm_daemon_coordinator_set_linger(cbm_daemon_coordinator_t *coordinator, bool linger) {
338+
if (!coordinator) {
339+
return;
340+
}
341+
cbm_mutex_lock(&coordinator->mutex);
342+
coordinator->linger = linger;
343+
/* Clearing the hold while the coordinator is already idle is the retirement
344+
* signal: an ephemeral generation whose cohort participants have all
345+
* connected and left (or whose linger window elapsed) transitions to
346+
* STOPPING now, exactly as the last-client disconnect would have. */
347+
if (!linger && !coordinator->permanent && coordinator->client_count == 0 &&
348+
coordinator->state == CBM_DAEMON_COORDINATOR_RUNNING) {
349+
coordinator->state = CBM_DAEMON_COORDINATOR_STOPPING;
350+
}
351+
cbm_mutex_unlock(&coordinator->mutex);
352+
}
353+
332354
cbm_daemon_coordinator_t *cbm_daemon_coordinator_new(uint64_t lease_timeout_ms) {
333355
cbm_daemon_coordinator_t *coordinator = calloc(1, sizeof(*coordinator));
334356
if (!coordinator) {

‎src/daemon/daemon.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,14 @@ cbm_daemon_coordinator_t *cbm_daemon_coordinator_new(uint64_t lease_timeout_ms);
7474
* self-transitions to STOPPING when its client count reaches zero; only the
7575
* explicit stop/drain paths end it. */
7676
void cbm_daemon_coordinator_set_permanent(cbm_daemon_coordinator_t *coordinator, bool permanent);
77+
/* Transient, runtime-driven hold (distinct from `permanent`) that keeps an
78+
* ephemeral coordinator RUNNING — still admitting new clients — after its last
79+
* client disconnects, while cohort participants are still mid-bootstrap racing
80+
* connect() (cold-storm race, 2026-09). Setting it before the last disconnect
81+
* suppresses the self-transition to STOPPING; clearing it while the coordinator
82+
* is already idle transitions to STOPPING at once, so retirement stays prompt
83+
* and bounded. Has no effect on a permanent coordinator. */
84+
void cbm_daemon_coordinator_set_linger(cbm_daemon_coordinator_t *coordinator, bool linger);
7785
/* The caller must first quiesce coordinator calls and hook invocations. */
7886
void cbm_daemon_coordinator_free(cbm_daemon_coordinator_t *coordinator);
7987

‎src/daemon/host.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -891,6 +891,10 @@ static bool host_wait_for_lifetime(cbm_daemon_runtime_service_t *service,
891891
return cbm_daemon_runtime_service_stop(service, HOST_RUNTIME_SHUTDOWN_MS);
892892
}
893893
host_http_reconcile_at(host, cbm_now_ms(), false);
894+
/* Retire an ephemeral generation that lingered for cold-storm cohort
895+
* participants once they drain, or once its bounded linger elapses.
896+
* A no-op unless a cohort-participant hook armed a linger. */
897+
cbm_daemon_runtime_service_reconcile_lifetime(service);
894898
(void)cbm_daemon_runtime_service_wait_exited(service, HOST_WAIT_TICK_MS);
895899
}
896900
}

‎src/daemon/runtime.c‎

Lines changed: 136 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,16 @@ void cbm_daemon_runtime_set_containment_hook_for_testing(
4949
cbm_daemon_runtime_containment_hook_t hook) {
5050
atomic_store(&runtime_containment_hook_seam, hook);
5151
}
52+
53+
/* Cold-storm ephemeral-linger seam (2026-09). Overrides the bounded linger
54+
* window that ephemeral last-committed-client retirement grants while cohort
55+
* participants are still mid-bootstrap, so a test can drive both the linger and
56+
* its expiry backstop in test time. UINT32_MAX leaves the production constant;
57+
* any other value (0 = expire immediately) overrides. */
58+
static _Atomic uint32_t runtime_ephemeral_linger_timeout_seam = UINT32_MAX;
59+
void cbm_daemon_runtime_service_set_ephemeral_linger_timeout_for_testing(uint32_t timeout_ms) {
60+
atomic_store(&runtime_ephemeral_linger_timeout_seam, timeout_ms);
61+
}
5262
#endif
5363

5464
#ifdef _WIN32
@@ -95,6 +105,14 @@ enum {
95105
* wedged behind it, and the dead generation held the endpoint pipes and
96106
* the UI port for nine hours with nothing logged. */
97107
RUNTIME_ABANDONED_REQUEST_JOIN_TIMEOUT_MS = 30000,
108+
/* Cold-storm race (2026-09): when the final committed client of an
109+
* ephemeral generation disconnects while cohort participants are still
110+
* admitted but mid-bootstrap (racing connect()), the generation lingers
111+
* this long for them to connect instead of retiring out from under them.
112+
* Mirrors the host initial-client window; the linger is always bounded so
113+
* a participant that never connects cannot wedge the generation (the
114+
* 900s host_serving hang). */
115+
RUNTIME_EPHEMERAL_LINGER_MS = 10000,
98116
RUNTIME_PATH_CAP = 4096,
99117

100118
RENDEZVOUS_REQUEST_ABI_OFFSET = 0,
@@ -231,6 +249,12 @@ struct cbm_daemon_runtime_service {
231249
/* Owned only by the convenience start() path. start_reserved() callers
232250
* retain their externally managed participant guard. */
233251
cbm_daemon_ipc_participant_guard_t *owned_participant_guard;
252+
/* Cold-storm ephemeral-retirement gate (2026-09). When the final committed
253+
* client disconnects while cohort participants are still admitted but not
254+
* yet committed, the generation lingers until this bounded deadline instead
255+
* of retiring; reconcile_lifetime retires it once that window elapses with
256+
* no new client committing. Zero means no linger is armed. */
257+
uint64_t ephemeral_linger_deadline_ms;
234258
};
235259

236260
struct cbm_daemon_runtime_worker {
@@ -289,6 +313,16 @@ static uint64_t runtime_deadline_after(uint32_t timeout_ms) {
289313
return now_ms + (uint64_t)timeout_ms;
290314
}
291315

316+
static uint32_t runtime_ephemeral_linger_timeout_ms(void) {
317+
#ifdef CBM_ENABLE_TEST_SEAMS
318+
uint32_t seam = atomic_load(&runtime_ephemeral_linger_timeout_seam);
319+
if (seam != UINT32_MAX) {
320+
return seam;
321+
}
322+
#endif
323+
return RUNTIME_EPHEMERAL_LINGER_MS;
324+
}
325+
292326
static void runtime_wait_tick(uint64_t deadline_ms) {
293327
uint64_t now_ms = cbm_now_ms();
294328
if (now_ms >= deadline_ms) {
@@ -1174,10 +1208,32 @@ static void runtime_service_interrupt_connections(cbm_daemon_runtime_service_t *
11741208
runtime_service_interrupt_connections_except(service, NULL, false);
11751209
}
11761210

1211+
/* A cold-storm racer is a connection that has been ACCEPTED but has not yet
1212+
* passed HELLO admission (in_use with a live connection, not yet admitted, not
1213+
* tearing down) -- a one-shot client still mid-bootstrap. This deliberately
1214+
* does NOT count an already-admitted provisional session (HELLO done, app
1215+
* session opening): a provisional coordinator client must not keep a retiring
1216+
* generation alive (see runtime_worker_disconnect below and the
1217+
* daemon_runtime_final_disconnect_rejects_blocked_provisional_session guard).
1218+
* Caller holds service->mutex. `except` excludes the departing worker. */
1219+
static bool runtime_has_pending_hello_peer_locked(const cbm_daemon_runtime_service_t *service,
1220+
const cbm_daemon_runtime_worker_t *except) {
1221+
for (size_t i = 0; i < service->worker_capacity; i++) {
1222+
const cbm_daemon_runtime_worker_t *worker = &service->workers[i];
1223+
if (worker != except && worker->in_use && worker->connection && !worker->admitted &&
1224+
!atomic_load_explicit(&worker->disconnecting, memory_order_acquire)) {
1225+
return true;
1226+
}
1227+
}
1228+
return false;
1229+
}
1230+
11771231
static void runtime_worker_disconnect(cbm_daemon_runtime_worker_t *worker) {
11781232
cbm_daemon_runtime_service_t *service = worker->service;
11791233
cbm_daemon_client_id_t client_id = CBM_DAEMON_CLIENT_ID_INVALID;
11801234
uint64_t shutdown_deadline = runtime_deadline_after(service->shutdown_timeout_ms);
1235+
bool last_committed_left = false;
1236+
bool linger_armed = false;
11811237
atomic_store_explicit(&worker->disconnecting, true, memory_order_release);
11821238
cbm_mutex_lock(&service->mutex);
11831239
if (worker->admitted) {
@@ -1195,14 +1251,47 @@ static void runtime_worker_disconnect(cbm_daemon_runtime_worker_t *worker) {
11951251
* alive after the final fully committed frontend disconnects.
11961252
* A permanent generation (`daemon start`) deliberately survives
11971253
* this: only the stop/drain ops or a process kill end it. */
1198-
runtime_service_begin_stopping_locked(service, shutdown_deadline, false,
1199-
"last_committed_client_disconnected");
1254+
last_committed_left = true;
1255+
/* Cold-storm race (2026-09): if another connection is already
1256+
* accepted and still mid-HELLO (not yet admitted) when the last
1257+
* committed client leaves, it is a one-shot peer racing to commit —
1258+
* retiring now strands it on a STOPPING daemon and forces a full
1259+
* cold respawn ("cold-storm client failed (racing daemon spawn)").
1260+
* Linger a bounded window for it; the accept loop retires the
1261+
* generation once that racer drains without committing or the window
1262+
* elapses, and a racer that does commit clears the linger
1263+
* (runtime_worker_commit_admission). This deliberately lingers ONLY
1264+
* for a pre-HELLO racer, never for an already-admitted provisional
1265+
* session (HELLO done, app session opening): the contract that a
1266+
* provisional coordinator client cannot keep a retiring generation
1267+
* alive is preserved (the reject-blocked-provisional-session guard).
1268+
* With no such racer — the common single-client case — retire
1269+
* immediately, unchanged. (A cross-process cohort peer count is
1270+
* unavailable: advisory locks expose presence, not a holder count,
1271+
* and Windows LockFileEx has no non-owning probe.) */
1272+
if (runtime_has_pending_hello_peer_locked(service, worker)) {
1273+
service->ephemeral_linger_deadline_ms =
1274+
runtime_deadline_after(runtime_ephemeral_linger_timeout_ms());
1275+
linger_armed = true;
1276+
} else {
1277+
service->ephemeral_linger_deadline_ms = 0;
1278+
runtime_service_begin_stopping_locked(service, shutdown_deadline, false,
1279+
"last_committed_client_disconnected");
1280+
}
12001281
}
12011282
}
12021283
cbm_mutex_unlock(&service->mutex);
12031284
if (client_id == CBM_DAEMON_CLIENT_ID_INVALID) {
12041285
return;
12051286
}
1287+
/* Set the coordinator hold BEFORE releasing this client so its last-client
1288+
* self-transition to STOPPING is suppressed while a racing peer is still
1289+
* mid-HELLO; when no peer is present the hold stays clear so the release
1290+
* retires the coordinator normally. last_committed_left implies a committed (hence
1291+
* admitted) client, so client_id is always valid here. */
1292+
if (last_committed_left) {
1293+
cbm_daemon_coordinator_set_linger(service->coordinator, linger_armed);
1294+
}
12061295
(void)cbm_daemon_client_disconnected(service->coordinator, client_id, cbm_now_ms());
12071296
if (worker->application_session_opened && !worker->application_cancelled) {
12081297
worker->application_cancelled = true;
@@ -1250,8 +1339,18 @@ static bool runtime_worker_commit_admission(cbm_daemon_runtime_worker_t *worker)
12501339
if (committed) {
12511340
worker->admission_committed = true;
12521341
service->committed_clients++;
1342+
/* A freshly committed client ends any last-client linger window; the
1343+
* next drop to zero re-arms it against the participants outstanding
1344+
* then. */
1345+
service->ephemeral_linger_deadline_ms = 0;
12531346
}
12541347
cbm_mutex_unlock(&service->mutex);
1348+
if (committed) {
1349+
/* Mirror the cleared linger onto the coordinator. client_count is
1350+
* already nonzero (admission incremented it), so this only resets the
1351+
* hold for the next idle window — it never retires a live coordinator. */
1352+
cbm_daemon_coordinator_set_linger(service->coordinator, false);
1353+
}
12551354
return committed;
12561355
}
12571356

@@ -2183,6 +2282,11 @@ static void *runtime_accept_loop(void *opaque) {
21832282
continue;
21842283
}
21852284

2285+
/* Self-retire a generation lingering for a cold-storm peer once the peer
2286+
* drains or the bounded window elapses, without an external driver. A
2287+
* no-op unless a last-committed-client linger is armed. */
2288+
cbm_daemon_runtime_service_reconcile_lifetime(service);
2289+
21862290
cbm_daemon_ipc_connection_t *connection = NULL;
21872291
int accepted =
21882292
cbm_daemon_ipc_accept(service->listener, RUNTIME_ACCEPT_POLL_MS, &connection);
@@ -2474,6 +2578,36 @@ size_t cbm_daemon_runtime_service_active_connections(cbm_daemon_runtime_service_
24742578
return count;
24752579
}
24762580

2581+
void cbm_daemon_runtime_service_reconcile_lifetime(cbm_daemon_runtime_service_t *service) {
2582+
if (!service) {
2583+
return;
2584+
}
2585+
bool retired = false;
2586+
cbm_mutex_lock(&service->mutex);
2587+
if (service->state == CBM_DAEMON_RUNTIME_SERVICE_RUNNING && !service->permanent &&
2588+
service->committed_clients == 0 && service->ephemeral_linger_deadline_ms != 0) {
2589+
bool peers_drained = !runtime_has_pending_hello_peer_locked(service, NULL);
2590+
bool window_elapsed = cbm_now_ms() >= service->ephemeral_linger_deadline_ms;
2591+
if (peers_drained || window_elapsed) {
2592+
/* The racing peer drained without committing (retire now), or never
2593+
* committed within the bounded window (the backstop that rules out
2594+
* an unbounded idle hang — the host_serving 900s mode). */
2595+
service->ephemeral_linger_deadline_ms = 0;
2596+
runtime_service_begin_stopping_locked(
2597+
service, runtime_deadline_after(service->shutdown_timeout_ms), false,
2598+
peers_drained ? "ephemeral_peer_drained" : "ephemeral_linger_expired");
2599+
retired = true;
2600+
}
2601+
}
2602+
cbm_mutex_unlock(&service->mutex);
2603+
if (retired) {
2604+
/* Release the coordinator hold now that the linger has resolved. With
2605+
* no client left this transitions the coordinator to STOPPING, so the
2606+
* service drains and exits cleanly rather than only on the deadline. */
2607+
cbm_daemon_coordinator_set_linger(service->coordinator, false);
2608+
}
2609+
}
2610+
24772611
size_t cbm_daemon_runtime_service_job_subscribers(cbm_daemon_runtime_service_t *service,
24782612
const char *project_key) {
24792613
return service ? cbm_daemon_job_subscribers(service->coordinator, project_key) : 0;

‎src/daemon/runtime.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,13 @@ uint64_t cbm_daemon_runtime_service_clients_admitted_total(cbm_daemon_runtime_se
329329
/* Includes accepted connections still waiting for HELLO. Never exceeds the
330330
* configured max_clients; over-cap peers receive REJECTED before close. */
331331
size_t cbm_daemon_runtime_service_active_connections(cbm_daemon_runtime_service_t *service);
332+
333+
/* Re-evaluate an ephemeral generation that is lingering after its last committed
334+
* client disconnected (the bounded cold-storm debounce). Retires it once the
335+
* linger window elapses with no new client committing; the bound also rules out
336+
* an unbounded idle hang. The host lifetime loop calls this every tick; it is a
337+
* no-op unless a last-committed-client linger is armed. */
338+
void cbm_daemon_runtime_service_reconcile_lifetime(cbm_daemon_runtime_service_t *service);
332339
size_t cbm_daemon_runtime_service_job_subscribers(cbm_daemon_runtime_service_t *service,
333340
const char *project_key);
334341
uint64_t cbm_daemon_runtime_service_client_process_id(cbm_daemon_runtime_service_t *service,
@@ -441,6 +448,12 @@ typedef void (*cbm_daemon_runtime_containment_hook_t)(const char *component);
441448
void cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(uint32_t timeout_ms);
442449
void cbm_daemon_runtime_set_containment_hook_for_testing(
443450
cbm_daemon_runtime_containment_hook_t hook);
451+
/* Cold-storm ephemeral-linger seam (2026-09). Overrides the bounded linger
452+
* window the last-committed-client retirement grants while cohort participants
453+
* are mid-bootstrap, so a test drives both the linger and its expiry backstop
454+
* in test time. UINT32_MAX restores the production constant; any other value
455+
* (0 = expire immediately) overrides. Process-global; reset it after use. */
456+
void cbm_daemon_runtime_service_set_ephemeral_linger_timeout_for_testing(uint32_t timeout_ms);
444457
#endif
445458

446459
#endif /* CBM_DAEMON_RUNTIME_H */

0 commit comments

Comments
 (0)