Skip to content

Commit 9e793df

Browse files
halindromeclaude
andcommitted
fix(daemon): keep held runtime files fresh and exit when one is lost
macOS com.apple.tmp_cleaner deletes /tmp files whose atime, mtime and ctime are all older than three days. The daemon's long-held lock files are empty and never written, so under a daemon that lives past three days the cleaner unlinks them while they are held. The next participant then creates and locks a fresh inode: forked index workers see the cohort daemon marker as free, read the live daemon as uncoordinated and exit 1 on every reindex, and a lost IPC lifetime lock or cohort lifetime file stops protecting the generation at all. A lost socket identity marker additionally makes listener close leave the socket pair behind. The host lifetime loop now runs a heartbeat every 60 s over every file the generation holds (cohort daemon claim and lease, participant guard, IPC lifetime reservation, identity marker). Each file is refreshed with futimens through its held descriptor, only after proving the path still names that descriptor, so the cleaner never selects it. If any file is already lost, the daemon logs daemon.lifetime_end reason=coordination_file_lost and stops cleanly; the next client starts a coordinated generation. Re-claiming in place would race that client. Windows needs no refresh (no age-based cleaner; held lock handles deny deletion), so its entry points only validate the handles. Closes DeusData#2178 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Signed-off-by: Shane McCarron <shane.mccarron@corvexconnect.com>
1 parent def38f3 commit 9e793df

11 files changed

Lines changed: 314 additions & 7 deletions

File tree

‎src/daemon/host.c‎

Lines changed: 55 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ enum {
5252
HOST_HTTP_RETRY_INITIAL_MS = 1000,
5353
HOST_HTTP_RETRY_MAX_MS = 30000,
5454
HOST_WATCH_INTERVAL_MS = 5000,
55+
/* Age-based temp cleaners need days of inactivity (macOS tmp_cleaner: 3);
56+
* a minute bounds the blind window after a long sleep (#2178). */
57+
HOST_COORDINATION_TOUCH_MS = 60000,
5558
HOST_CONFLICT_LOG_CAP = 1024 * 1024,
5659
HOST_OPERATION_LOG_CAP = 5 * 1024 * 1024,
5760
HOST_PATH_CAP = 4096,
@@ -868,10 +871,42 @@ static bool host_background_start(host_state_t *host) {
868871
return true;
869872
}
870873

874+
/* Coordination handles the host retains for the generation's whole lifetime. */
875+
typedef struct {
876+
const cbm_daemon_ipc_endpoint_t *endpoint;
877+
cbm_version_cohort_lease_t *cohort_lease;
878+
cbm_version_cohort_daemon_claim_t *daemon_claim;
879+
cbm_daemon_ipc_participant_guard_t *participant_guard;
880+
} host_coordination_t;
881+
882+
/* Refresh every long-held runtime file and name the first one found lost, or
883+
* return NULL. A held lock whose path now names another inode coordinates
884+
* nothing: forked index workers read the daemon as uncoordinated and a peer
885+
* can start a second generation (#2178). */
886+
static const char *host_coordination_touch(const host_coordination_t *coordination,
887+
cbm_daemon_runtime_service_t *service) {
888+
if (!cbm_version_cohort_daemon_claim_touch(coordination->daemon_claim)) {
889+
return "cohort_daemon_claim";
890+
}
891+
if (!cbm_version_cohort_lease_touch(coordination->cohort_lease)) {
892+
return "cohort_lease";
893+
}
894+
if (!cbm_daemon_ipc_participant_guard_touch(coordination->endpoint,
895+
coordination->participant_guard)) {
896+
return "participant_guard";
897+
}
898+
if (!cbm_daemon_runtime_service_touch_listener(service)) {
899+
return "listener";
900+
}
901+
return NULL;
902+
}
903+
871904
static bool host_wait_for_lifetime(cbm_daemon_runtime_service_t *service,
872-
atomic_int *stop_requested, host_state_t *host, bool permanent) {
905+
atomic_int *stop_requested, host_state_t *host, bool permanent,
906+
const host_coordination_t *coordination) {
873907
uint64_t initial_deadline = cbm_now_ms() + HOST_INITIAL_CLIENT_TIMEOUT_MS;
874908
uint64_t stopping_deadline = 0;
909+
uint64_t next_touch = 0;
875910
for (;;) {
876911
cbm_daemon_runtime_service_state_t state = cbm_daemon_runtime_service_state(service);
877912
if (state == CBM_DAEMON_RUNTIME_SERVICE_EXITED) {
@@ -903,6 +938,17 @@ static bool host_wait_for_lifetime(cbm_daemon_runtime_service_t *service,
903938
stop ? "stop_requested" : "initial_window_expired");
904939
return cbm_daemon_runtime_service_stop(service, HOST_RUNTIME_SHUTDOWN_MS);
905940
}
941+
/* Permanent generations too: exiting lets the next client start a
942+
* coordinated generation, while re-claiming would race that client. */
943+
if (cbm_now_ms() >= next_touch) {
944+
next_touch = cbm_now_ms() + HOST_COORDINATION_TOUCH_MS;
945+
const char *lost = host_coordination_touch(coordination, service);
946+
if (lost) {
947+
cbm_log_warn("daemon.lifetime_end", "reason", "coordination_file_lost", "file",
948+
lost);
949+
return cbm_daemon_runtime_service_stop(service, HOST_RUNTIME_SHUTDOWN_MS);
950+
}
951+
}
906952
host_http_reconcile_at(host, cbm_now_ms(), false);
907953
/* Retire an ephemeral generation that lingered for cold-storm cohort
908954
* participants once they drain, or once its bounded linger elapses.
@@ -1123,7 +1169,14 @@ int cbm_daemon_host_run(const cbm_daemon_host_config_t *config) {
11231169
: "unavailable",
11241170
"memory_budget_bytes", memory_budget, "physical_job_limit", physical_job_limit,
11251171
"worker_memory_budget_bytes", worker_memory_budget);
1126-
if (!host_wait_for_lifetime(service, config->stop_requested, &host, config->permanent)) {
1172+
host_coordination_t coordination = {
1173+
.endpoint = config->endpoint,
1174+
.cohort_lease = cohort_lease,
1175+
.daemon_claim = daemon_claim,
1176+
.participant_guard = participant_guard,
1177+
};
1178+
if (!host_wait_for_lifetime(service, config->stop_requested, &host, config->permanent,
1179+
&coordination)) {
11271180
host_force_terminate("runtime");
11281181
}
11291182

‎src/daemon/ipc.c‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3220,6 +3220,46 @@ void cbm_daemon_ipc_listener_close(cbm_daemon_ipc_listener_t *listener) {
32203220
free(listener);
32213221
}
32223222

3223+
/* Touch through the held descriptor only after proving the path still names
3224+
* it: refreshing by path would keep a replacement inode fresh and hide the
3225+
* loss (#2178). */
3226+
static bool posix_held_file_touch(int directory_fd, const char *base_name, int fd) {
3227+
return private_regular_file_snapshot(directory_fd, base_name, fd, 1, NULL) &&
3228+
futimens(fd, NULL) == 0;
3229+
}
3230+
3231+
static bool posix_held_lock_touch(int directory_fd, const process_lock_entry_t *entry, int fd) {
3232+
return entry && posix_held_file_touch(directory_fd, entry->lock_name, fd);
3233+
}
3234+
3235+
bool cbm_daemon_ipc_listener_touch(cbm_daemon_ipc_listener_t *listener) {
3236+
if (!listener || listener->dir_fd < 0 || listener->owner_pid != getpid() ||
3237+
!listener->lifetime_reservation) {
3238+
return false;
3239+
}
3240+
bool lifetime =
3241+
posix_held_lock_touch(listener->dir_fd, listener->lifetime_reservation->process_entry,
3242+
listener->lifetime_reservation->fd);
3243+
const cbm_daemon_ipc_participant_guard_t *guard = listener->participant_guard;
3244+
bool participant =
3245+
!guard ||
3246+
posix_held_lock_touch(listener->dir_fd, guard->legacy_process_entry, guard->legacy_fd);
3247+
/* The marker is not held open; reopen it and require the published inode. */
3248+
int marker_fd = openat(listener->dir_fd, listener->socket_identity_name,
3249+
O_RDONLY | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK);
3250+
struct stat marker_status;
3251+
bool marker = marker_fd >= 0 &&
3252+
private_regular_file_snapshot(listener->dir_fd, listener->socket_identity_name,
3253+
marker_fd, 1, &marker_status) &&
3254+
marker_status.st_dev == listener->identity_device &&
3255+
marker_status.st_ino == listener->identity_inode &&
3256+
futimens(marker_fd, NULL) == 0;
3257+
if (marker_fd >= 0) {
3258+
(void)close(marker_fd);
3259+
}
3260+
return lifetime && participant && marker;
3261+
}
3262+
32233263
int cbm_daemon_ipc_accept(cbm_daemon_ipc_listener_t *listener, uint32_t timeout_ms,
32243264
cbm_daemon_ipc_connection_t **connection_out) {
32253265
if (connection_out) {
@@ -3603,6 +3643,12 @@ bool cbm_daemon_ipc_participant_guard_release(cbm_daemon_ipc_participant_guard_t
36033643
return true;
36043644
}
36053645

3646+
bool cbm_daemon_ipc_participant_guard_touch(const cbm_daemon_ipc_endpoint_t *endpoint,
3647+
cbm_daemon_ipc_participant_guard_t *guard) {
3648+
return endpoint && guard && guard->owner_pid == getpid() &&
3649+
posix_held_lock_touch(endpoint->dir_fd, guard->legacy_process_entry, guard->legacy_fd);
3650+
}
3651+
36063652
int cbm_daemon_ipc_local_transition_try_acquire(
36073653
const cbm_daemon_ipc_endpoint_t *endpoint, cbm_daemon_ipc_local_transition_t **transition_out) {
36083654
if (transition_out) {
@@ -6746,6 +6792,17 @@ bool cbm_daemon_ipc_participant_guard_release(cbm_daemon_ipc_participant_guard_t
67466792
return true;
67476793
}
67486794

6795+
/* Windows has no age-based cleaner of the private runtime directory, and its
6796+
* held handles deny deletion; the #2178 heartbeat has nothing to refresh. */
6797+
bool cbm_daemon_ipc_listener_touch(cbm_daemon_ipc_listener_t *listener) {
6798+
return listener != NULL;
6799+
}
6800+
6801+
bool cbm_daemon_ipc_participant_guard_touch(const cbm_daemon_ipc_endpoint_t *endpoint,
6802+
cbm_daemon_ipc_participant_guard_t *guard) {
6803+
return endpoint && guard;
6804+
}
6805+
67496806
int cbm_daemon_ipc_local_transition_try_acquire(
67506807
const cbm_daemon_ipc_endpoint_t *endpoint, cbm_daemon_ipc_local_transition_t **transition_out) {
67516808
if (transition_out) {

‎src/daemon/ipc.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,15 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
7676
cbm_daemon_ipc_lifetime_reservation_t **reservation_io);
7777
void cbm_daemon_ipc_listener_close(cbm_daemon_ipc_listener_t *listener);
7878

79+
/* Heartbeat for the daemon's long-lived runtime artifacts: refresh the held
80+
* lifetime reservation, the listener's participant guard (if it owns one) and
81+
* the published identity marker so an age-based temp cleaner never deletes
82+
* them (#2178). Returns false once any of them no longer names the file this
83+
* listener owns: a lost lifetime file lets a peer start a second generation,
84+
* and a lost identity marker makes listener close leave the socket pair
85+
* behind. Only the owning thread may call it while the listener is open. */
86+
bool cbm_daemon_ipc_listener_touch(cbm_daemon_ipc_listener_t *listener);
87+
7988
/* Create or validate one private current-user directory at an already
8089
* canonical local path. Ancestors are handle-validated without mutation:
8190
* POSIX permits only root/current-user owners and no group/other write (apart
@@ -270,6 +279,10 @@ bool cbm_daemon_ipc_startup_lock_release(cbm_daemon_ipc_startup_lock_t **lock_io
270279
int cbm_daemon_ipc_participant_guard_try_join(const cbm_daemon_ipc_endpoint_t *endpoint,
271280
cbm_daemon_ipc_participant_guard_t **guard_out);
272281
bool cbm_daemon_ipc_participant_guard_release(cbm_daemon_ipc_participant_guard_t **guard_io);
282+
/* Same heartbeat contract as cbm_daemon_ipc_listener_touch, for a guard the
283+
* caller retains itself. */
284+
bool cbm_daemon_ipc_participant_guard_touch(const cbm_daemon_ipc_endpoint_t *endpoint,
285+
cbm_daemon_ipc_participant_guard_t *guard);
273286

274287
/* Standalone CLI work joins the legacy-compatible current participant group
275288
* without becoming a daemon client. Acquisition retains startup-v2 only for

‎src/daemon/runtime.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2578,6 +2578,10 @@ size_t cbm_daemon_runtime_service_active_connections(cbm_daemon_runtime_service_
25782578
return count;
25792579
}
25802580

2581+
bool cbm_daemon_runtime_service_touch_listener(cbm_daemon_runtime_service_t *service) {
2582+
return service && service->listener && cbm_daemon_ipc_listener_touch(service->listener);
2583+
}
2584+
25812585
void cbm_daemon_runtime_service_reconcile_lifetime(cbm_daemon_runtime_service_t *service) {
25822586
if (!service) {
25832587
return;

‎src/daemon/runtime.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,10 @@ size_t cbm_daemon_runtime_service_active_connections(cbm_daemon_runtime_service_
336336
* an unbounded idle hang. The host lifetime loop calls this every tick; it is a
337337
* no-op unless a last-committed-client linger is armed. */
338338
void cbm_daemon_runtime_service_reconcile_lifetime(cbm_daemon_runtime_service_t *service);
339+
/* #2178 heartbeat over the listener's runtime artifacts; see
340+
* cbm_daemon_ipc_listener_touch. Call only from the thread that stops and
341+
* frees the service, which is the only thread that closes the listener. */
342+
bool cbm_daemon_runtime_service_touch_listener(cbm_daemon_runtime_service_t *service);
339343
size_t cbm_daemon_runtime_service_job_subscribers(cbm_daemon_runtime_service_t *service,
340344
const char *project_key);
341345
uint64_t cbm_daemon_runtime_service_client_process_id(cbm_daemon_runtime_service_t *service,

‎src/daemon/version_cohort.c‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,25 @@ cbm_private_file_lock_status_t cbm_version_cohort_lease_release(
363363
return result;
364364
}
365365

366+
static bool version_cohort_lock_touch(cbm_private_file_lock_t *lock) {
367+
return !lock || cbm_private_file_lock_touch(lock) == CBM_PRIVATE_FILE_LOCK_OK;
368+
}
369+
370+
bool cbm_version_cohort_lease_touch(cbm_version_cohort_lease_t *lease) {
371+
if (!lease) {
372+
return false;
373+
}
374+
/* Evaluate all three so one lost file does not leave the others to age. */
375+
bool lifetime = version_cohort_lock_touch(lease->lifetime);
376+
bool admission = version_cohort_lock_touch(lease->admission);
377+
bool maintenance = version_cohort_lock_touch(lease->maintenance);
378+
return lifetime && admission && maintenance;
379+
}
380+
381+
bool cbm_version_cohort_daemon_claim_touch(cbm_version_cohort_daemon_claim_t *claim) {
382+
return claim && claim->marker && version_cohort_lock_touch(claim->marker);
383+
}
384+
366385
static cbm_version_cohort_status_t version_cohort_failed(cbm_version_cohort_lease_t *lease,
367386
cbm_version_cohort_status_t status,
368387
cbm_version_cohort_lease_t **lease_out) {

‎src/daemon/version_cohort.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,15 @@ cbm_version_cohort_daemon_presence_t cbm_version_cohort_daemon_presence_under_tr
145145
cbm_private_file_lock_status_t cbm_version_cohort_lease_release(
146146
cbm_version_cohort_lease_t **lease_io);
147147

148+
/* Heartbeat for a long-lived holder: refresh every lock file the lease or
149+
* claim holds and report false once any held file was unlinked or replaced.
150+
* An age-based temp cleaner deleting a held file silently ends coordination,
151+
* because the next participant creates and locks a fresh inode (#2178). The
152+
* only safe response to false is an orderly exit; re-claiming would race
153+
* that participant. */
154+
bool cbm_version_cohort_lease_touch(cbm_version_cohort_lease_t *lease);
155+
bool cbm_version_cohort_daemon_claim_touch(cbm_version_cohort_daemon_claim_t *claim);
156+
148157
/* Refuses teardown while a lease or retryable cleanup handle remains. */
149158
cbm_private_file_lock_status_t cbm_version_cohort_manager_free(
150159
cbm_version_cohort_manager_t **manager_io);

‎src/foundation/private_file_lock.c‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -482,6 +482,24 @@ cbm_private_file_lock_status_t cbm_private_file_lock_payload_write(cbm_private_f
482482
return valid ? CBM_PRIVATE_FILE_LOCK_OK : CBM_PRIVATE_FILE_LOCK_IO;
483483
}
484484

485+
cbm_private_file_lock_status_t cbm_private_file_lock_touch(cbm_private_file_lock_t *lock) {
486+
if (!lock) {
487+
return CBM_PRIVATE_FILE_LOCK_UNSAFE;
488+
}
489+
if (!cbm_private_file_lock_fork_guard_enter()) {
490+
return CBM_PRIVATE_FILE_LOCK_IO;
491+
}
492+
/* An unlinked file keeps its inode alive for this handle with st_nlink 0,
493+
* which the payload validity check already rejects. */
494+
bool linked = private_payload_fd_valid(lock, NULL);
495+
bool touched = linked && futimens(lock->fd, NULL) == 0;
496+
cbm_private_file_lock_fork_guard_leave();
497+
if (!linked) {
498+
return CBM_PRIVATE_FILE_LOCK_UNSAFE;
499+
}
500+
return touched ? CBM_PRIVATE_FILE_LOCK_OK : CBM_PRIVATE_FILE_LOCK_IO;
501+
}
502+
485503
cbm_private_file_lock_status_t cbm_private_file_lock_release(cbm_private_file_lock_t **lock_io) {
486504
if (!lock_io || !*lock_io) {
487505
return CBM_PRIVATE_FILE_LOCK_IO;
@@ -1450,6 +1468,20 @@ cbm_private_file_lock_status_t cbm_private_file_lock_payload_write(cbm_private_f
14501468
return valid ? CBM_PRIVATE_FILE_LOCK_OK : CBM_PRIVATE_FILE_LOCK_IO;
14511469
}
14521470

1471+
cbm_private_file_lock_status_t cbm_private_file_lock_touch(cbm_private_file_lock_t *lock) {
1472+
/* Windows has no age-based cleaner of the private lock directory; only
1473+
* confirm the held handle still names a linked file. */
1474+
if (!lock) {
1475+
return CBM_PRIVATE_FILE_LOCK_UNSAFE;
1476+
}
1477+
if (!cbm_private_file_lock_fork_guard_enter()) {
1478+
return CBM_PRIVATE_FILE_LOCK_IO;
1479+
}
1480+
bool linked = private_win_payload_handle_valid(lock);
1481+
cbm_private_file_lock_fork_guard_leave();
1482+
return linked ? CBM_PRIVATE_FILE_LOCK_OK : CBM_PRIVATE_FILE_LOCK_UNSAFE;
1483+
}
1484+
14531485
static bool private_win_release_unlock(cbm_private_file_lock_t *lock) {
14541486
lock->test_unlock_attempts++;
14551487
if (lock->test_fail_unlock_once) {

‎src/foundation/private_file_lock_internal.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,13 @@ cbm_private_file_lock_status_t cbm_private_file_lock_payload_write(cbm_private_f
6161
const void *buffer,
6262
size_t length);
6363

64+
/* Long-lived holders call this periodically. It refreshes the held file's
65+
* timestamps through the validated handle so age-based temp cleaners (macOS
66+
* tmp_cleaner, systemd-tmpfiles) never select it, and returns UNSAFE once the
67+
* handle no longer names a linked file: a peer opening the path would then
68+
* lock a different inode and the lock coordinates nothing (#2178). */
69+
cbm_private_file_lock_status_t cbm_private_file_lock_touch(cbm_private_file_lock_t *lock);
70+
6471
/* Forces the next successfully acquired native lock down the post-lock
6572
* validation cleanup path and injects pre-call release failures there. */
6673
bool cbm_private_lock_directory_fail_post_acquire_cleanup_for_test(

0 commit comments

Comments
 (0)