Skip to content

Commit 5e9f146

Browse files
authored
Merge pull request DeusData#2289 from halindrome/fix/cohort-lock-tmp-cleaner
fix(daemon): keep held runtime files fresh and exit when one is lost
2 parents 07e73a3 + f2dad9a commit 5e9f146

14 files changed

Lines changed: 783 additions & 3 deletions

‎src/daemon/host.c‎

Lines changed: 98 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ enum {
5252
HOST_HTTP_RETRY_INITIAL_MS = 1000,
5353
HOST_HTTP_RETRY_MAX_MS = 30000,
5454
HOST_WATCH_INTERVAL_MS = 5000,
55+
/* Age-based temp cleaners need days of inactivity (macOS tmp_cleaner: 3);
56+
* a minute bounds the blind window after a long sleep (#2178). */
57+
HOST_COORDINATION_TOUCH_MS = 60000,
5558
HOST_CONFLICT_LOG_CAP = 1024 * 1024,
5659
HOST_OPERATION_LOG_CAP = 5 * 1024 * 1024,
5760
HOST_PATH_CAP = 4096,
@@ -875,10 +878,58 @@ static bool host_background_start(host_state_t *host) {
875878
return true;
876879
}
877880

881+
/* Coordination handles the host retains for the generation's whole lifetime. */
882+
typedef struct {
883+
const cbm_daemon_ipc_endpoint_t *endpoint;
884+
cbm_version_cohort_lease_t *cohort_lease;
885+
cbm_version_cohort_daemon_claim_t *daemon_claim;
886+
cbm_daemon_ipc_participant_guard_t *participant_guard;
887+
} host_coordination_t;
888+
889+
typedef struct {
890+
const char *lost; /* first held file whose path no longer names it */
891+
const char *transient; /* first still-valid file that could not be refreshed */
892+
} host_touch_result_t;
893+
894+
static void host_touch_note(host_touch_result_t *result, const char *file, int status) {
895+
if (status == 0 && !result->lost) {
896+
result->lost = file;
897+
} else if (status < 0 && !result->transient) {
898+
result->transient = file;
899+
}
900+
}
901+
902+
static int host_cohort_touch_status(cbm_version_cohort_status_t status) {
903+
return status == CBM_VERSION_COHORT_OK ? 1 : (status == CBM_VERSION_COHORT_IO ? -1 : 0);
904+
}
905+
906+
/* Refresh every long-held runtime file. A held lock whose path now names
907+
* another inode coordinates nothing: forked index workers read the daemon as
908+
* uncoordinated and a peer can start a second generation (#2178). Every file
909+
* is visited even after one fails so the rest keep being refreshed. */
910+
static host_touch_result_t host_coordination_touch(const host_coordination_t *coordination,
911+
cbm_daemon_runtime_service_t *service) {
912+
host_touch_result_t result = {0};
913+
host_touch_note(&result, "cohort_daemon_claim",
914+
host_cohort_touch_status(
915+
cbm_version_cohort_daemon_claim_touch(coordination->daemon_claim)));
916+
host_touch_note(
917+
&result, "cohort_lease",
918+
host_cohort_touch_status(cbm_version_cohort_lease_touch(coordination->cohort_lease)));
919+
host_touch_note(&result, "participant_guard",
920+
cbm_daemon_ipc_participant_guard_touch(coordination->endpoint,
921+
coordination->participant_guard));
922+
host_touch_note(&result, "listener", cbm_daemon_runtime_service_touch_listener(service));
923+
return result;
924+
}
925+
878926
static bool host_wait_for_lifetime(cbm_daemon_runtime_service_t *service,
879-
atomic_int *stop_requested, host_state_t *host, bool permanent) {
927+
atomic_int *stop_requested, host_state_t *host, bool permanent,
928+
const host_coordination_t *coordination) {
880929
uint64_t initial_deadline = cbm_now_ms() + HOST_INITIAL_CLIENT_TIMEOUT_MS;
881930
uint64_t stopping_deadline = 0;
931+
uint64_t next_touch = 0;
932+
bool touch_degraded = false;
882933
for (;;) {
883934
cbm_daemon_runtime_service_state_t state = cbm_daemon_runtime_service_state(service);
884935
if (state == CBM_DAEMON_RUNTIME_SERVICE_EXITED) {
@@ -910,6 +961,28 @@ static bool host_wait_for_lifetime(cbm_daemon_runtime_service_t *service,
910961
stop ? "stop_requested" : "initial_window_expired");
911962
return cbm_daemon_runtime_service_stop(service, HOST_RUNTIME_SHUTDOWN_MS);
912963
}
964+
/* Permanent generations too: exiting lets the next client start a
965+
* coordinated generation, while re-claiming would race that client. */
966+
if (cbm_now_ms() >= next_touch) {
967+
next_touch = cbm_now_ms() + HOST_COORDINATION_TOUCH_MS;
968+
host_touch_result_t touched = host_coordination_touch(coordination, service);
969+
if (touched.lost) {
970+
cbm_log_warn("daemon.lifetime_end", "reason", "coordination_file_lost", "file",
971+
touched.lost);
972+
return cbm_daemon_runtime_service_stop(service, HOST_RUNTIME_SHUTDOWN_MS);
973+
}
974+
/* A refresh failure on a still-valid file (read-only remount,
975+
* EMFILE) is not loss: keep serving, retry next tick, and log
976+
* only transitions so a persistent condition cannot flood. */
977+
if ((touched.transient != NULL) != touch_degraded) {
978+
touch_degraded = touched.transient != NULL;
979+
if (touch_degraded) {
980+
cbm_log_warn("daemon.coordination_touch_failed", "file", touched.transient);
981+
} else {
982+
cbm_log_info("daemon.coordination_touch_recovered");
983+
}
984+
}
985+
}
913986
host_http_reconcile_at(host, cbm_now_ms(), false);
914987
/* Retire an ephemeral generation that lingered for cold-storm cohort
915988
* participants once they drain, or once its bounded linger elapses.
@@ -919,6 +992,22 @@ static bool host_wait_for_lifetime(cbm_daemon_runtime_service_t *service,
919992
}
920993
}
921994

995+
bool cbm_daemon_host_wait_for_lifetime_for_test(cbm_daemon_runtime_service_t *service,
996+
const cbm_daemon_ipc_endpoint_t *endpoint,
997+
cbm_version_cohort_lease_t *cohort_lease,
998+
cbm_version_cohort_daemon_claim_t *daemon_claim,
999+
cbm_daemon_ipc_participant_guard_t *guard) {
1000+
host_state_t host = {0};
1001+
atomic_int stop_requested = 0;
1002+
host_coordination_t coordination = {
1003+
.endpoint = endpoint,
1004+
.cohort_lease = cohort_lease,
1005+
.daemon_claim = daemon_claim,
1006+
.participant_guard = guard,
1007+
};
1008+
return host_wait_for_lifetime(service, &stop_requested, &host, false, &coordination);
1009+
}
1010+
9221011
static bool host_application_shutdown(host_state_t *host) {
9231012
if (cbm_daemon_application_shutdown(host->application, HOST_APPLICATION_SHUTDOWN_MS)) {
9241013
return true;
@@ -1130,7 +1219,14 @@ int cbm_daemon_host_run(const cbm_daemon_host_config_t *config) {
11301219
: "unavailable",
11311220
"memory_budget_bytes", memory_budget, "physical_job_limit", physical_job_limit,
11321221
"worker_memory_budget_bytes", worker_memory_budget);
1133-
if (!host_wait_for_lifetime(service, config->stop_requested, &host, config->permanent)) {
1222+
host_coordination_t coordination = {
1223+
.endpoint = config->endpoint,
1224+
.cohort_lease = cohort_lease,
1225+
.daemon_claim = daemon_claim,
1226+
.participant_guard = participant_guard,
1227+
};
1228+
if (!host_wait_for_lifetime(service, config->stop_requested, &host, config->permanent,
1229+
&coordination)) {
11341230
host_force_terminate("runtime");
11351231
}
11361232

‎src/daemon/host_internal.h‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@
99
#include <stdint.h>
1010

1111
struct cbm_daemon_ipc_endpoint;
12+
struct cbm_daemon_ipc_participant_guard;
13+
struct cbm_daemon_runtime_service;
14+
struct cbm_version_cohort_daemon_claim;
15+
struct cbm_version_cohort_lease;
1216

1317
typedef bool (*cbm_daemon_host_cleanup_release_for_test_fn)(void *context);
1418

@@ -68,4 +72,14 @@ bool cbm_daemon_host_http_reconcile_free_refusal_for_test(
6872
* the failure, the adapter must cancel so the final free succeeds. */
6973
bool cbm_daemon_host_http_thread_create_failure_lifecycle_for_test(void);
7074

75+
/* Run the production lifetime loop over caller-held coordination handles with
76+
* no HTTP host state and no stop request. The generation is ephemeral, so a
77+
* loop that misses a loss still ends at the initial client window instead of
78+
* hanging the test. Returns the loop's own result. */
79+
bool cbm_daemon_host_wait_for_lifetime_for_test(
80+
struct cbm_daemon_runtime_service *service, const struct cbm_daemon_ipc_endpoint *endpoint,
81+
struct cbm_version_cohort_lease *cohort_lease,
82+
struct cbm_version_cohort_daemon_claim *daemon_claim,
83+
struct cbm_daemon_ipc_participant_guard *guard);
84+
7185
#endif /* CBM_DAEMON_HOST_INTERNAL_H */

‎src/daemon/ipc.c‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3510,6 +3510,94 @@ void cbm_daemon_ipc_listener_close(cbm_daemon_ipc_listener_t *listener) {
35103510
free(listener);
35113511
}
35123512

3513+
/* Touch results: 1 refreshed, 0 lost (the path no longer names the owned
3514+
* file), -1 transient (a still-valid file could not be refreshed). Touch
3515+
* through the held descriptor only after proving the path still names it:
3516+
* refreshing by path would keep a replacement inode fresh and hide the loss
3517+
* (#2178). */
3518+
static int posix_touch_fold(int result, int next) {
3519+
return result == 0 || next == 0 ? 0 : (result < 0 || next < 0 ? -1 : 1);
3520+
}
3521+
3522+
/* Whether base_name still names the regular file (device, inode): 1 yes,
3523+
* 0 proven not (absent, another inode, a symlink or other non-regular
3524+
* entry), -1 undetermined (any other lookup failure, e.g. EIO or ESTALE). */
3525+
static int posix_path_names_inode(int directory_fd, const char *base_name, dev_t device,
3526+
ino_t inode) {
3527+
struct stat by_path;
3528+
if (fstatat(directory_fd, base_name, &by_path, AT_SYMLINK_NOFOLLOW) != 0) {
3529+
return errno == ENOENT ? 0 : -1;
3530+
}
3531+
return S_ISREG(by_path.st_mode) && by_path.st_dev == device && by_path.st_ino == inode ? 1 : 0;
3532+
}
3533+
3534+
static int posix_held_file_touch(int directory_fd, const char *base_name, int fd) {
3535+
struct stat by_handle;
3536+
if (fstat(fd, &by_handle) != 0) {
3537+
return -1;
3538+
}
3539+
int identity =
3540+
by_handle.st_nlink == 0
3541+
? 0
3542+
: posix_path_names_inode(directory_fd, base_name, by_handle.st_dev, by_handle.st_ino);
3543+
if (identity <= 0) {
3544+
return identity;
3545+
}
3546+
return futimens(fd, NULL) == 0 ? 1 : -1;
3547+
}
3548+
3549+
static int posix_held_lock_touch(int directory_fd, const process_lock_entry_t *entry, int fd) {
3550+
return entry ? posix_held_file_touch(directory_fd, entry->lock_name, fd) : 0;
3551+
}
3552+
3553+
static int posix_identity_marker_touch(const cbm_daemon_ipc_listener_t *listener) {
3554+
/* The marker is not held open. Classify by path first, so a symlink or
3555+
* other replacement is loss; only then reopen it to refresh, where a
3556+
* failure (EMFILE, EACCES) says nothing about the file and is transient. */
3557+
int identity = posix_path_names_inode(listener->dir_fd, listener->socket_identity_name,
3558+
listener->identity_device, listener->identity_inode);
3559+
if (identity <= 0) {
3560+
return identity;
3561+
}
3562+
int marker_fd = openat(listener->dir_fd, listener->socket_identity_name,
3563+
O_RDONLY | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK);
3564+
if (marker_fd < 0) {
3565+
return errno == ENOENT ? 0 : -1;
3566+
}
3567+
struct stat marker_status;
3568+
int result = -1;
3569+
if (fstat(marker_fd, &marker_status) == 0) {
3570+
/* Replaced between the lookup and the open. */
3571+
result = marker_status.st_dev != listener->identity_device ||
3572+
marker_status.st_ino != listener->identity_inode
3573+
? 0
3574+
: (futimens(marker_fd, NULL) == 0 ? 1 : -1);
3575+
}
3576+
(void)close(marker_fd);
3577+
return result;
3578+
}
3579+
3580+
int cbm_daemon_ipc_listener_touch(cbm_daemon_ipc_listener_t *listener,
3581+
cbm_daemon_ipc_participant_guard_t *external_guard) {
3582+
if (!listener || listener->dir_fd < 0 || listener->owner_pid != getpid() ||
3583+
!listener->lifetime_reservation) {
3584+
return 0;
3585+
}
3586+
int result =
3587+
posix_held_lock_touch(listener->dir_fd, listener->lifetime_reservation->process_entry,
3588+
listener->lifetime_reservation->fd);
3589+
const cbm_daemon_ipc_participant_guard_t *guards[] = {listener->participant_guard,
3590+
external_guard};
3591+
for (size_t i = 0; i < sizeof(guards) / sizeof(guards[0]); i++) {
3592+
if (guards[i]) {
3593+
result = posix_touch_fold(result, posix_held_lock_touch(listener->dir_fd,
3594+
guards[i]->legacy_process_entry,
3595+
guards[i]->legacy_fd));
3596+
}
3597+
}
3598+
return posix_touch_fold(result, posix_identity_marker_touch(listener));
3599+
}
3600+
35133601
int cbm_daemon_ipc_accept(cbm_daemon_ipc_listener_t *listener, uint32_t timeout_ms,
35143602
cbm_daemon_ipc_connection_t **connection_out) {
35153603
if (connection_out) {
@@ -3893,6 +3981,14 @@ bool cbm_daemon_ipc_participant_guard_release(cbm_daemon_ipc_participant_guard_t
38933981
return true;
38943982
}
38953983

3984+
int cbm_daemon_ipc_participant_guard_touch(const cbm_daemon_ipc_endpoint_t *endpoint,
3985+
cbm_daemon_ipc_participant_guard_t *guard) {
3986+
if (!endpoint || !guard || guard->owner_pid != getpid()) {
3987+
return 0;
3988+
}
3989+
return posix_held_lock_touch(endpoint->dir_fd, guard->legacy_process_entry, guard->legacy_fd);
3990+
}
3991+
38963992
int cbm_daemon_ipc_local_transition_try_acquire(
38973993
const cbm_daemon_ipc_endpoint_t *endpoint, cbm_daemon_ipc_local_transition_t **transition_out) {
38983994
if (transition_out) {
@@ -7090,6 +7186,19 @@ bool cbm_daemon_ipc_participant_guard_release(cbm_daemon_ipc_participant_guard_t
70907186
return true;
70917187
}
70927188

7189+
/* Windows has no age-based cleaner of the private runtime directory, and its
7190+
* held handles deny deletion; the #2178 heartbeat has nothing to refresh. */
7191+
int cbm_daemon_ipc_listener_touch(cbm_daemon_ipc_listener_t *listener,
7192+
cbm_daemon_ipc_participant_guard_t *external_guard) {
7193+
(void)external_guard;
7194+
return listener ? 1 : 0;
7195+
}
7196+
7197+
int cbm_daemon_ipc_participant_guard_touch(const cbm_daemon_ipc_endpoint_t *endpoint,
7198+
cbm_daemon_ipc_participant_guard_t *guard) {
7199+
return endpoint && guard ? 1 : 0;
7200+
}
7201+
70937202
int cbm_daemon_ipc_local_transition_try_acquire(
70947203
const cbm_daemon_ipc_endpoint_t *endpoint, cbm_daemon_ipc_local_transition_t **transition_out) {
70957204
if (transition_out) {

‎src/daemon/ipc.h‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,19 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
7676
cbm_daemon_ipc_lifetime_reservation_t **reservation_io);
7777
void cbm_daemon_ipc_listener_close(cbm_daemon_ipc_listener_t *listener);
7878

79+
/* Heartbeat for the daemon's long-lived runtime artifacts: refresh the held
80+
* lifetime reservation, the listener's participant guard (if it owns one),
81+
* external_guard (optional, a guard the caller retains beside this listener)
82+
* and the published identity marker so an age-based temp cleaner never
83+
* deletes them (#2178). Returns 1 when all were refreshed, 0 once any no
84+
* longer names the file this listener owns (a lost lifetime file lets a peer
85+
* start a second generation; a lost identity marker makes listener close
86+
* leave the socket pair behind), and -1 when a still-valid file could not be
87+
* refreshed (transient). Only the thread that closes the listener may call it
88+
* while the listener is open. */
89+
int cbm_daemon_ipc_listener_touch(cbm_daemon_ipc_listener_t *listener,
90+
cbm_daemon_ipc_participant_guard_t *external_guard);
91+
7992
/* Create or validate one private current-user directory at an already
8093
* canonical local path. Ancestors are handle-validated without mutation:
8194
* POSIX permits only root/current-user owners and no group/other write (apart
@@ -290,6 +303,10 @@ bool cbm_daemon_ipc_startup_lock_release(cbm_daemon_ipc_startup_lock_t **lock_io
290303
int cbm_daemon_ipc_participant_guard_try_join(const cbm_daemon_ipc_endpoint_t *endpoint,
291304
cbm_daemon_ipc_participant_guard_t **guard_out);
292305
bool cbm_daemon_ipc_participant_guard_release(cbm_daemon_ipc_participant_guard_t **guard_io);
306+
/* Same heartbeat contract and 1/0/-1 result as cbm_daemon_ipc_listener_touch,
307+
* for a guard the caller retains itself. */
308+
int cbm_daemon_ipc_participant_guard_touch(const cbm_daemon_ipc_endpoint_t *endpoint,
309+
cbm_daemon_ipc_participant_guard_t *guard);
293310

294311
/* Standalone CLI work joins the legacy-compatible current participant group
295312
* without becoming a daemon client. Acquisition retains startup-v2 only for

‎src/daemon/runtime.c‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2724,6 +2724,12 @@ size_t cbm_daemon_runtime_service_active_connections(cbm_daemon_runtime_service_
27242724
return count;
27252725
}
27262726

2727+
int cbm_daemon_runtime_service_touch_listener(cbm_daemon_runtime_service_t *service) {
2728+
return service && service->listener
2729+
? cbm_daemon_ipc_listener_touch(service->listener, service->owned_participant_guard)
2730+
: 0;
2731+
}
2732+
27272733
void cbm_daemon_runtime_service_reconcile_lifetime(cbm_daemon_runtime_service_t *service) {
27282734
if (!service) {
27292735
return;

‎src/daemon/runtime.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,12 @@ size_t cbm_daemon_runtime_service_active_connections(cbm_daemon_runtime_service_
347347
* an unbounded idle hang. The host lifetime loop calls this every tick; it is a
348348
* no-op unless a last-committed-client linger is armed. */
349349
void cbm_daemon_runtime_service_reconcile_lifetime(cbm_daemon_runtime_service_t *service);
350+
/* #2178 heartbeat over the listener's runtime artifacts, plus the participant
351+
* guard the service owns when it started itself; see
352+
* cbm_daemon_ipc_listener_touch for the 1/0/-1 result. Call only from the
353+
* thread that stops and frees the service, which is the only thread that
354+
* closes the listener. */
355+
int cbm_daemon_runtime_service_touch_listener(cbm_daemon_runtime_service_t *service);
350356
size_t cbm_daemon_runtime_service_job_subscribers(cbm_daemon_runtime_service_t *service,
351357
const char *project_key);
352358
uint64_t cbm_daemon_runtime_service_client_process_id(cbm_daemon_runtime_service_t *service,

‎src/daemon/version_cohort.c‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,37 @@ cbm_private_file_lock_status_t cbm_version_cohort_lease_release(
363363
return result;
364364
}
365365

366+
/* Folds one lock's touch into an accumulated result: a lost file (UNSAFE)
367+
* outranks a transient refresh failure (IO), which outranks OK. A lease holds
368+
* only lifetime after admission, so absent locks are skipped. */
369+
static cbm_version_cohort_status_t version_cohort_lock_touch(cbm_private_file_lock_t *lock,
370+
cbm_version_cohort_status_t result) {
371+
if (!lock || result == CBM_VERSION_COHORT_UNSAFE) {
372+
return result;
373+
}
374+
cbm_private_file_lock_status_t status = cbm_private_file_lock_touch(lock);
375+
if (status == CBM_PRIVATE_FILE_LOCK_UNSAFE) {
376+
return CBM_VERSION_COHORT_UNSAFE;
377+
}
378+
return status == CBM_PRIVATE_FILE_LOCK_OK ? result : CBM_VERSION_COHORT_IO;
379+
}
380+
381+
cbm_version_cohort_status_t cbm_version_cohort_lease_touch(cbm_version_cohort_lease_t *lease) {
382+
if (!lease || !lease->lifetime) {
383+
return CBM_VERSION_COHORT_UNSAFE;
384+
}
385+
cbm_version_cohort_status_t result =
386+
version_cohort_lock_touch(lease->lifetime, CBM_VERSION_COHORT_OK);
387+
result = version_cohort_lock_touch(lease->admission, result);
388+
return version_cohort_lock_touch(lease->maintenance, result);
389+
}
390+
391+
cbm_version_cohort_status_t cbm_version_cohort_daemon_claim_touch(
392+
cbm_version_cohort_daemon_claim_t *claim) {
393+
return claim && claim->marker ? version_cohort_lock_touch(claim->marker, CBM_VERSION_COHORT_OK)
394+
: CBM_VERSION_COHORT_UNSAFE;
395+
}
396+
366397
static cbm_version_cohort_status_t version_cohort_failed(cbm_version_cohort_lease_t *lease,
367398
cbm_version_cohort_status_t status,
368399
cbm_version_cohort_lease_t **lease_out) {

0 commit comments

Comments
 (0)