Skip to content

Commit 33c4fd8

Browse files
DeusDataclaude
andcommitted
fix(daemon): name errno and path in daemon.ipc.listen_failed
A full /tmp made every daemon start die at pending publication, and the only durable trace was `daemon.ipc.listen_failed stage=pending_publication` -- no syscall, no errno, no path. The reporter needed hours (and a `df` on the wrong mount) to find the cause (DeusData#1828). Every listener failure now records the stage, the errno the failing step reported, and the artifact path it was operating on, and the log line carries all three: daemon.ipc.listen_failed stage=pending_publication errno=ENOSPC path=/tmp/cbm-daemon-1000/cbm-<key>.pending.tmp The record publication helper notes which step refused (create, write, link, unlink, sync, close, read-back) and on which artifact, so the listener can name it. The bind failure keeps its errno but joins the same shape (stage=socket_bind errno=... path=...) instead of a bare numeric bind_errno. cbm_errno_name() maps the portable errno set to its symbol. cbm_daemon_ipc_listen_failure_detail() exposes the same triple to the daemon host for the fail-fast record that follows. Test: daemon_ipc_listen_failure_names_errno_and_path forces the record write to fail with ENOSPC through a narrow inert seam (the socket binds, the file is created, only its data write is refused -- the reporter's exact shape) and asserts the log line names the errno and the temp path, and that the namespace is left clean. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 29dbf0b commit 33c4fd8

6 files changed

Lines changed: 310 additions & 17 deletions

File tree

‎src/daemon/ipc.c‎

Lines changed: 144 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include "foundation/compat_thread.h"
1010
#include "foundation/log.h"
1111
#include "foundation/macos_acl.h"
12+
#include "foundation/platform.h"
1213
#include "foundation/private_file_lock_internal.h"
1314
#include "foundation/sha256.h"
1415
#include "foundation/secure_random.h"
@@ -53,6 +54,53 @@ const char *cbm_daemon_ipc_validation_detail(void) {
5354
return ipc_validation_detail_buffer;
5455
}
5556

57+
static cbm_daemon_ipc_listen_failure_t ipc_listen_failure;
58+
59+
static void ipc_listen_failure_reset(void) {
60+
memset(&ipc_listen_failure, 0, sizeof(ipc_listen_failure));
61+
}
62+
63+
/* Record and log one listener failure. `name` is an artifact inside the
64+
* endpoint's runtime directory, "" for the directory itself, or NULL when
65+
* the stage has no path. The log line names the errno symbolically and the
66+
* full path, so `daemon.ipc.listen_failed stage=pending_publication` alone
67+
* (the whole trace #1828's reporter had) can no longer happen. */
68+
static void ipc_listen_failed(const char *runtime_dir, const char *stage, int errno_value,
69+
const char *name) {
70+
ipc_listen_failure_reset();
71+
(void)snprintf(ipc_listen_failure.stage, sizeof(ipc_listen_failure.stage), "%s",
72+
stage ? stage : "");
73+
ipc_listen_failure.errno_value = errno_value;
74+
if (runtime_dir && name) {
75+
int written = name[0] ? snprintf(ipc_listen_failure.path, sizeof(ipc_listen_failure.path),
76+
"%s/%s", runtime_dir, name)
77+
: snprintf(ipc_listen_failure.path, sizeof(ipc_listen_failure.path),
78+
"%s", runtime_dir);
79+
if (written <= 0 || written >= (int)sizeof(ipc_listen_failure.path)) {
80+
ipc_listen_failure.path[0] = '\0';
81+
}
82+
}
83+
const char *errno_name = errno_value != 0 ? cbm_errno_name(errno_value) : NULL;
84+
if (errno_name && ipc_listen_failure.path[0]) {
85+
cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "errno", errno_name, "path",
86+
ipc_listen_failure.path);
87+
} else if (errno_name) {
88+
cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "errno", errno_name);
89+
} else if (ipc_listen_failure.path[0]) {
90+
cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "path", ipc_listen_failure.path);
91+
} else {
92+
cbm_log_error("daemon.ipc.listen_failed", "stage", stage);
93+
}
94+
}
95+
96+
bool cbm_daemon_ipc_listen_failure_detail(cbm_daemon_ipc_listen_failure_t *out) {
97+
if (!out) {
98+
return false;
99+
}
100+
*out = ipc_listen_failure;
101+
return ipc_listen_failure.stage[0] != '\0';
102+
}
103+
56104
static bool instance_key_valid(const char *key) {
57105
if (!key) {
58106
return false;
@@ -122,6 +170,19 @@ void cbm_daemon_ipc_windows_legacy_guard_release_failures_set_for_test(unsigned
122170
memory_order_release);
123171
}
124172

173+
#ifndef _WIN32
174+
static atomic_int g_posix_record_write_failure_errno_for_test;
175+
#endif
176+
177+
void cbm_daemon_ipc_posix_record_write_failure_set_for_test(int errno_value) {
178+
#ifndef _WIN32
179+
atomic_store_explicit(&g_posix_record_write_failure_errno_for_test, errno_value,
180+
memory_order_release);
181+
#else
182+
(void)errno_value;
183+
#endif
184+
}
185+
125186
#ifdef _WIN32
126187
static cbm_daemon_ipc_startup_gate_fn g_startup_gate_for_test;
127188
static void *g_startup_gate_context_for_test;
@@ -1815,6 +1876,29 @@ static bool posix_fd_write_all(int fd, const uint8_t *buffer, size_t length) {
18151876
return true;
18161877
}
18171878

1879+
/* The failing step of the most recent record publication in this process:
1880+
* its errno (0 when a validation predicate, not a syscall, refused) and the
1881+
* artifact name it was operating on. Read by the listener right after a
1882+
* publish returns false. */
1883+
static int posix_publish_failure_errno;
1884+
static char posix_publish_failure_name[NAME_MAX + 1];
1885+
1886+
static void posix_publish_failure_note(int errno_value, const char *name) {
1887+
posix_publish_failure_errno = errno_value;
1888+
(void)snprintf(posix_publish_failure_name, sizeof(posix_publish_failure_name), "%s",
1889+
name ? name : "");
1890+
}
1891+
1892+
static bool posix_record_fd_write(int fd, const uint8_t *buffer, size_t length) {
1893+
int injected =
1894+
atomic_load_explicit(&g_posix_record_write_failure_errno_for_test, memory_order_acquire);
1895+
if (injected != 0) {
1896+
errno = injected;
1897+
return false;
1898+
}
1899+
return posix_fd_write_all(fd, buffer, length);
1900+
}
1901+
18181902
static bool posix_fd_pread_all(int fd, uint8_t *buffer, size_t length) {
18191903
size_t offset = 0;
18201904
while (offset < length) {
@@ -2169,50 +2253,75 @@ static bool posix_socket_record_publish(const cbm_daemon_ipc_endpoint_t *endpoin
21692253
return false;
21702254
}
21712255
struct stat existing;
2256+
errno = 0;
21722257
if (fstatat(endpoint->dir_fd, record_name, &existing, AT_SYMLINK_NOFOLLOW) == 0 ||
21732258
errno != ENOENT) {
2259+
posix_publish_failure_note(errno == 0 ? EEXIST : errno, record_name);
21742260
return false;
21752261
}
21762262
uint8_t record[POSIX_SOCKET_RECORD_SIZE];
21772263
if (!posix_socket_record_encode(magic, source, record)) {
2264+
posix_publish_failure_note(0, record_name);
21782265
return false;
21792266
}
21802267

21812268
char temp_name[NAME_MAX + 1];
21822269
if (!posix_socket_record_temp_name(record_name, temp_name)) {
2270+
posix_publish_failure_note(ENAMETOOLONG, record_name);
21832271
return false;
21842272
}
21852273
int fd = openat(endpoint->dir_fd, temp_name,
21862274
O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK, 0600);
21872275
if (fd < 0) {
2276+
posix_publish_failure_note(errno, temp_name);
21882277
return false;
21892278
}
21902279

21912280
struct stat created = {0};
21922281
bool temp_exists = true;
21932282
bool stable_linked = false;
2283+
errno = 0;
21942284
bool ok = fd_set_cloexec(fd) && fchmod(fd, 0600) == 0 &&
21952285
private_regular_file_snapshot(endpoint->dir_fd, temp_name, fd, 1, &created) &&
2196-
posix_fd_write_all(fd, record, sizeof(record)) && fsync(fd) == 0 &&
2286+
posix_record_fd_write(fd, record, sizeof(record)) && fsync(fd) == 0 &&
21972287
private_regular_file_snapshot(endpoint->dir_fd, temp_name, fd, 1, &created) &&
21982288
created.st_size == (off_t)POSIX_SOCKET_RECORD_SIZE &&
21992289
endpoint_runtime_still_valid(endpoint);
2290+
if (!ok) {
2291+
posix_publish_failure_note(errno, temp_name);
2292+
}
22002293
if (ok) {
22012294
posix_record_publication_stage_reached(magic, false);
2295+
errno = 0;
22022296
ok =
22032297
posix_linkat_no_follow(endpoint->dir_fd, temp_name, endpoint->dir_fd, record_name) == 0;
22042298
stable_linked = ok;
2299+
if (!ok) {
2300+
posix_publish_failure_note(errno, record_name);
2301+
}
22052302
}
22062303
if (ok) {
22072304
posix_record_publication_stage_reached(magic, true);
2305+
errno = 0;
22082306
ok = posix_path_unlink_regular_if_matches(endpoint->dir_fd, temp_name, created.st_dev,
22092307
created.st_ino, 2);
22102308
temp_exists = !ok;
2309+
if (!ok) {
2310+
posix_publish_failure_note(errno, temp_name);
2311+
}
22112312
}
22122313
if (ok) {
2314+
errno = 0;
22132315
ok = posix_directory_sync(endpoint->dir_fd);
2316+
if (!ok) {
2317+
posix_publish_failure_note(errno, "");
2318+
}
22142319
}
2320+
errno = 0;
22152321
if (close(fd) != 0) {
2322+
if (ok) {
2323+
posix_publish_failure_note(errno, temp_name);
2324+
}
22162325
ok = false;
22172326
}
22182327

@@ -2224,6 +2333,9 @@ static bool posix_socket_record_publish(const cbm_daemon_ipc_endpoint_t *endpoin
22242333
posix_socket_identity_equal(&published_record.identity, &source->identity) &&
22252334
strcmp(published_record.anchor_name, source->anchor_name) == 0 &&
22262335
published_status.st_dev == created.st_dev && published_status.st_ino == created.st_ino;
2336+
if (!ok) {
2337+
posix_publish_failure_note(0, record_name);
2338+
}
22272339
}
22282340
if (!ok) {
22292341
if (stable_linked) {
@@ -2539,8 +2651,11 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
25392651
cbm_daemon_ipc_lifetime_reservation_t **reservation_io) {
25402652
cbm_daemon_ipc_lifetime_reservation_t *lifetime_reservation =
25412653
reservation_io ? *reservation_io : NULL;
2654+
ipc_listen_failure_reset();
2655+
posix_publish_failure_note(0, NULL);
2656+
const char *runtime_dir = endpoint ? endpoint->runtime_dir : NULL;
25422657
if (!lifetime_reservation_matches_endpoint(endpoint, lifetime_reservation)) {
2543-
cbm_log_error("daemon.ipc.listen_failed", "stage", "reservation_validation");
2658+
ipc_listen_failed(runtime_dir, "reservation_validation", 0, NULL);
25442659
return NULL;
25452660
}
25462661
/* Stale removal happens only under the startup lock, before a daemon host
@@ -2550,7 +2665,7 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
25502665
char pending_temp_name[NAME_MAX + 1];
25512666
if (!posix_socket_record_temp_name(endpoint->socket_identity_name, identity_temp_name) ||
25522667
!posix_socket_record_temp_name(endpoint->socket_pending_name, pending_temp_name)) {
2553-
cbm_log_error("daemon.ipc.listen_failed", "stage", "temp_names");
2668+
ipc_listen_failed(runtime_dir, "temp_names", ENAMETOOLONG, endpoint->socket_pending_name);
25542669
return NULL;
25552670
}
25562671
struct stat existing;
@@ -2559,27 +2674,32 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
25592674
endpoint->socket_pending_name, identity_temp_name, pending_temp_name,
25602675
};
25612676
bool namespace_absent = true;
2677+
const char *occupied_name = "";
2678+
int occupied_errno = 0;
25622679
for (size_t index = 0; index < sizeof(required_absent) / sizeof(required_absent[0]); index++) {
2680+
errno = 0;
25632681
if (fstatat(endpoint->dir_fd, required_absent[index], &existing, AT_SYMLINK_NOFOLLOW) ==
25642682
0 ||
25652683
errno != ENOENT) {
25662684
namespace_absent = false;
2685+
occupied_name = required_absent[index];
2686+
occupied_errno = errno == 0 ? EEXIST : errno;
25672687
break;
25682688
}
25692689
}
25702690
if (!endpoint_runtime_still_valid(endpoint) || !namespace_absent) {
2571-
cbm_log_error("daemon.ipc.listen_failed", "stage", "namespace_validation");
2691+
ipc_listen_failed(runtime_dir, "namespace_validation", occupied_errno, occupied_name);
25722692
return NULL;
25732693
}
25742694

25752695
int fd = local_socket_new();
25762696
if (fd < 0) {
2577-
cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_creation");
2697+
ipc_listen_failed(runtime_dir, "socket_creation", errno, NULL);
25782698
return NULL;
25792699
}
25802700
cbm_daemon_ipc_listener_t *listener = calloc(1, sizeof(*listener));
25812701
if (!listener) {
2582-
cbm_log_error("daemon.ipc.listen_failed", "stage", "listener_allocation");
2702+
ipc_listen_failed(runtime_dir, "listener_allocation", ENOMEM, NULL);
25832703
(void)close(fd);
25842704
return NULL;
25852705
}
@@ -2597,7 +2717,8 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
25972717
if (listener->dir_fd < 0 || !fd_set_cloexec(listener->dir_fd) || !listener->runtime_dir ||
25982718
!listener->address || !listener->socket_name || !listener->socket_anchor_name ||
25992719
!listener->socket_identity_name || !listener->socket_pending_name) {
2600-
cbm_log_error("daemon.ipc.listen_failed", "stage", "listener_initialization");
2720+
ipc_listen_failed(runtime_dir, "listener_initialization",
2721+
listener->dir_fd < 0 ? errno : ENOMEM, NULL);
26012722
if (listener->dir_fd >= 0) {
26022723
(void)close(listener->dir_fd);
26032724
}
@@ -2615,20 +2736,19 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
26152736
struct sockaddr_un address;
26162737
socklen_t address_length;
26172738
if (!unix_address_set(&address, endpoint->socket_anchor_address, &address_length)) {
2618-
cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_address");
2739+
ipc_listen_failed(runtime_dir, "socket_address", ENAMETOOLONG,
2740+
endpoint->socket_anchor_name);
26192741
cbm_daemon_ipc_listener_close(listener);
26202742
return NULL;
26212743
}
26222744
if (bind(fd, (const struct sockaddr *)&address, address_length) != 0) {
2623-
int bind_error = errno;
2624-
char error_text[32];
2625-
(void)snprintf(error_text, sizeof(error_text), "%d", bind_error);
2626-
cbm_log_error("daemon.ipc.listen_failed", "bind_errno", error_text);
2745+
ipc_listen_failed(runtime_dir, "socket_bind", errno, endpoint->socket_anchor_name);
26272746
cbm_daemon_ipc_listener_close(listener);
26282747
return NULL;
26292748
}
26302749

26312750
struct stat bound_status;
2751+
errno = 0;
26322752
bool bound_path_ok = fstatat(endpoint->dir_fd, endpoint->socket_anchor_name, &bound_status,
26332753
AT_SYMLINK_NOFOLLOW) == 0 &&
26342754
S_ISSOCK(bound_status.st_mode) && bound_status.st_uid == geteuid();
@@ -2648,11 +2768,13 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
26482768
anchor_status.st_ino != bound_status.st_ino ||
26492769
!posix_socket_identity_from_stat(&anchor_status, &anchor_identity) ||
26502770
!posix_directory_sync(endpoint->dir_fd)) {
2771+
int security_errno = errno;
26512772
if (bound_path_ok) {
26522773
posix_bound_socket_unlink_if_matches(endpoint->dir_fd, endpoint->socket_anchor_name,
26532774
bound_status.st_dev, bound_status.st_ino);
26542775
}
2655-
cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_security");
2776+
ipc_listen_failed(runtime_dir, "socket_security", security_errno,
2777+
endpoint->socket_anchor_name);
26562778
cbm_daemon_ipc_listener_close(listener);
26572779
return NULL;
26582780
}
@@ -2669,16 +2791,19 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
26692791
endpoint, endpoint->socket_pending_name, POSIX_SOCKET_PENDING_MAGIC, &pending,
26702792
&pending_status.st_dev, &pending_status.st_ino);
26712793
if (!pending_published) {
2672-
cbm_log_error("daemon.ipc.listen_failed", "stage", "pending_publication");
2794+
ipc_listen_failed(runtime_dir, "pending_publication", posix_publish_failure_errno,
2795+
posix_publish_failure_name);
26732796
posix_publication_abort(endpoint, &anchor_identity, false, NULL, false, NULL);
26742797
cbm_daemon_ipc_listener_close(listener);
26752798
return NULL;
26762799
}
26772800
posix_publication_stage_reached(CBM_DAEMON_IPC_POSIX_PUBLICATION_PENDING_DURABLE);
26782801

2802+
errno = 0;
26792803
bool stable_linked = posix_linkat_no_follow(endpoint->dir_fd, endpoint->socket_anchor_name,
26802804
endpoint->dir_fd, endpoint->socket_name) == 0 &&
26812805
posix_directory_sync(endpoint->dir_fd);
2806+
int stable_errno = stable_linked ? 0 : errno;
26822807
posix_socket_identity_t stable_identity = {0};
26832808
posix_socket_identity_t committed_identity = {0};
26842809
struct stat stable_status = {0};
@@ -2693,7 +2818,7 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
26932818
posix_socket_identity_equal(&stable_identity, &committed_identity) &&
26942819
posix_socket_inode_equal(&anchor_identity, &committed_identity);
26952820
if (!stable_valid) {
2696-
cbm_log_error("daemon.ipc.listen_failed", "stage", "stable_publication");
2821+
ipc_listen_failed(runtime_dir, "stable_publication", stable_errno, endpoint->socket_name);
26972822
posix_publication_abort(endpoint, &anchor_identity, pending_published, &pending_status,
26982823
false, NULL);
26992824
cbm_daemon_ipc_listener_close(listener);
@@ -2712,7 +2837,8 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
27122837
endpoint, endpoint->socket_identity_name, POSIX_SOCKET_MARKER_MAGIC, &marker,
27132838
&marker_status.st_dev, &marker_status.st_ino);
27142839
if (!marker_published) {
2715-
cbm_log_error("daemon.ipc.listen_failed", "stage", "marker_publication");
2840+
ipc_listen_failed(runtime_dir, "marker_publication", posix_publish_failure_errno,
2841+
posix_publish_failure_name);
27162842
posix_publication_abort(endpoint, &committed_identity, pending_published, &pending_status,
27172843
false, NULL);
27182844
cbm_daemon_ipc_listener_close(listener);
@@ -2722,10 +2848,11 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
27222848
listener->identity_inode = marker_status.st_ino;
27232849
posix_publication_stage_reached(CBM_DAEMON_IPC_POSIX_PUBLICATION_MARKER_DURABLE);
27242850

2851+
errno = 0;
27252852
if (!posix_path_unlink_regular_if_matches(endpoint->dir_fd, endpoint->socket_pending_name,
27262853
pending_status.st_dev, pending_status.st_ino, 1) ||
27272854
!posix_directory_sync(endpoint->dir_fd)) {
2728-
cbm_log_error("daemon.ipc.listen_failed", "stage", "pending_removal");
2855+
ipc_listen_failed(runtime_dir, "pending_removal", errno, endpoint->socket_pending_name);
27292856
posix_publication_abort(endpoint, &committed_identity, true, &pending_status,
27302857
marker_published, &marker_status);
27312858
cbm_daemon_ipc_listener_close(listener);

‎src/daemon/ipc.h‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,25 @@ bool cbm_daemon_ipc_private_directory_secure(const char *directory_path);
9191
* validation refusal (empty string when none). Diagnostic only — callers
9292
* append it to their error messages; policy decisions never read it. */
9393
const char *cbm_daemon_ipc_validation_detail(void);
94+
95+
/* This process's most recent listener-publication failure: the stage that
96+
* refused, the errno the failing step reported (0 when the step reported
97+
* none), and the artifact path it was operating on ("" when no path applies).
98+
* Every `daemon.ipc.listen_failed` log line carries the same three fields.
99+
* Reset at the start of each listen attempt. Returns false while no failure
100+
* has been recorded. Diagnostic only: the daemon host copies it into its
101+
* durable start-failure record so a waiting client can name the cause instead
102+
* of burning its full startup deadline (#1828). */
103+
enum {
104+
CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP = 32,
105+
CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP = 4096
106+
};
107+
typedef struct {
108+
char stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP];
109+
int errno_value;
110+
char path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP];
111+
} cbm_daemon_ipc_listen_failure_t;
112+
bool cbm_daemon_ipc_listen_failure_detail(cbm_daemon_ipc_listen_failure_t *out);
94113
#ifdef CBM_ENABLE_TEST_SEAMS
95114
/* #1537: seed the detail so a test can prove the CLI refusal surfaces it. */
96115
void cbm_daemon_ipc_set_validation_detail_for_testing(const char *detail);

‎src/daemon/ipc_internal.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,11 @@ typedef void (*cbm_daemon_ipc_posix_publication_hook_fn)(
9595
void cbm_daemon_ipc_posix_publication_hook_set_for_test(
9696
cbm_daemon_ipc_posix_publication_hook_fn hook, void *context);
9797
void cbm_daemon_ipc_windows_legacy_guard_release_failures_set_for_test(unsigned int count);
98+
/* #1828: while non-zero, every POSIX socket-record publication write fails
99+
* with this errno. This is the deterministic stand-in for a full runtime
100+
* filesystem (tmpfs ENOSPC): the socket binds, the record file is created,
101+
* and only its data write is refused -- exactly the reporter's failure shape. */
102+
void cbm_daemon_ipc_posix_record_write_failure_set_for_test(int errno_value);
98103

99104
/* Deterministic-interleaving seam: fires on the Windows startup path once the
100105
* startup lock is held, before the rendezvous handoff. A test parks here to

0 commit comments

Comments
 (0)