Skip to content

Commit 91e3121

Browse files
DeusDataclaude
andcommitted
fix(pipeline): take the stage lock before the stage file is visible (close DeusData#2111 create→register race)
create_staging_path() minted the stage's main file with mkstemp and only then took its sidecar .lock via stage_owner_register(). A second, concurrent cbm_pipeline_run() against the same final_path (auto_index, or the stale-rendezvous-recovery retry path) could land its sweep_orphan_stages() in that window: the main file was visible but not yet locked, so the sweep treated it like a genuinely orphaned stage and deleted it out from under the live writer (POSIX), or collided on the not-yet-created sidecar and surfaced EACCES/errno=13 (Windows). That is the DeusData#2111 red on test-windows-guards and both test-windows CLANG64 x86_64 shards (index_format / simhash / index_resilience, with pipeline.stage action=lock_failed errno=13). Fix: mint the six-char alphanumeric suffix in create_staging_path() itself, create and hold the sidecar lock FIRST, then create the main file with O_EXCL. The stage's main file is therefore never present on disk without its lock already held, so a racing sweep can only ever find the stage lock-held: it reaches the stage through the .lock entry too (stage_entry_stage_length() matches it), probes the lock, sees a live holder, and keeps it. The create->register TOCTOU is closed by construction. DeusData#1839 is preserved: sweep_one_stage() still removes a stage whose sidecar is absent (ENOENT). With live stages now always carrying their sidecar, ENOENT is only ever a genuine pre-lock-era orphan minted by an older binary, which DeusData#1839 requires be swept. stage_owner_register() becomes stage_owner_adopt(), recording the already-held descriptor; re-taking the lock would self-conflict, since flock() and Windows _SH_DENYRW deny a second acquire of the same sidecar even from this process. A new after_stage_created test seam and pipeline_concurrent_sweep_must_not_- remove_inflight_stage drive a real concurrent sweep in that window; the test is RED without the lock-first ordering. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 7b4eb4a commit 91e3121

3 files changed

Lines changed: 258 additions & 28 deletions

File tree

‎src/pipeline/pipeline.c‎

Lines changed: 123 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 };
3737
#include "foundation/compat_thread.h"
3838
#include "foundation/profile.h"
3939
#include "foundation/mem.h"
40+
#include "foundation/secure_random.h"
4041

4142
#include <ctype.h>
4243
#include <errno.h>
@@ -73,6 +74,8 @@ static atomic_bool g_persist_test_cancel_after_destination_prepare = false;
7374
static atomic_bool g_persist_test_fail_adr_capture = false;
7475
static cbm_pipeline_test_hook_fn g_persist_test_before_final_manifest = NULL;
7576
static void *g_persist_test_before_final_manifest_userdata = NULL;
77+
static cbm_pipeline_test_hook_fn g_persist_test_after_stage_created = NULL;
78+
static void *g_persist_test_after_stage_created_userdata = NULL;
7679

7780
void cbm_pipeline_incremental_test_fail_after_stage_dump_once(void) {
7881
atomic_store(&g_persist_test_fail_after_stage_dump, true);
@@ -106,6 +109,29 @@ void cbm_pipeline_persist_test_run_before_final_manifest(void) {
106109
}
107110
}
108111

112+
void cbm_pipeline_incremental_test_after_stage_created_once(cbm_pipeline_test_hook_fn hook,
113+
void *userdata) {
114+
g_persist_test_after_stage_created = hook;
115+
g_persist_test_after_stage_created_userdata = userdata;
116+
}
117+
118+
/* Fired by create_staging_path() right after the stage's main file is created
119+
* with O_EXCL -- and, in the current lock-before-visible ordering, after its
120+
* sidecar lock is already held. A test hook installed here can run a
121+
* concurrent sweep (another cbm_pipeline_run() against the same final_path) at
122+
* this instant to prove the just-created stage survives it. Under the OLD
123+
* create-then-lock ordering this was the unlocked TOCTOU window, so the same
124+
* hook binds RED if that ordering ever regresses. */
125+
void cbm_pipeline_persist_test_run_after_stage_created(void) {
126+
cbm_pipeline_test_hook_fn hook = g_persist_test_after_stage_created;
127+
void *userdata = g_persist_test_after_stage_created_userdata;
128+
g_persist_test_after_stage_created = NULL;
129+
g_persist_test_after_stage_created_userdata = NULL;
130+
if (hook) {
131+
hook(userdata);
132+
}
133+
}
134+
109135
bool cbm_pipeline_persist_test_take_failure_after_stage_dump(void) {
110136
return atomic_exchange(&g_persist_test_fail_after_stage_dump, false);
111137
}
@@ -125,6 +151,8 @@ void cbm_pipeline_persist_test_reset_faults(void) {
125151
atomic_store(&g_persist_test_fail_adr_capture, false);
126152
g_persist_test_before_final_manifest = NULL;
127153
g_persist_test_before_final_manifest_userdata = NULL;
154+
g_persist_test_after_stage_created = NULL;
155+
g_persist_test_after_stage_created_userdata = NULL;
128156
}
129157
#endif
130158

@@ -1630,25 +1658,25 @@ void cbm_pipeline_stage_lock_drop(const char *stage_path, int lock_fd) {
16301658
}
16311659
}
16321660

1633-
static bool stage_owner_register(const char *stage_path) {
1634-
int fd = cbm_pipeline_stage_lock_hold(stage_path);
1635-
if (fd < 0) {
1636-
char errno_text[16];
1637-
(void)snprintf(errno_text, sizeof(errno_text), "%d", errno);
1638-
cbm_log_warn("pipeline.stage", "action", "lock_failed", "errno", errno_text, "path",
1639-
stage_path);
1640-
return false;
1641-
}
1661+
/* Record an already-held stage lock in the per-process owner table, keyed by
1662+
* path so publish/finalize/discard can release it later. On success the table
1663+
* owns lock_fd; on failure the caller still does and must drop it.
1664+
*
1665+
* The lock must ALREADY be held: create_staging_path() takes it before the
1666+
* stage's main file is created (so the file is never visible on disk without
1667+
* its lock), then hands the descriptor here. Re-taking the lock in this helper
1668+
* would self-conflict -- both flock() and Windows _SH_DENYRW deny a second
1669+
* acquire of the same sidecar even from this same process. */
1670+
static bool stage_owner_adopt(const char *stage_path, int lock_fd) {
16421671
stage_owner_t *owner = (stage_owner_t *)malloc(sizeof(*owner));
16431672
char *path_copy = strdup(stage_path);
16441673
if (!owner || !path_copy) {
16451674
free(owner);
16461675
free(path_copy);
1647-
cbm_pipeline_stage_lock_drop(stage_path, fd);
16481676
return false;
16491677
}
16501678
owner->stage_path = path_copy;
1651-
owner->lock_fd = fd;
1679+
owner->lock_fd = lock_fd;
16521680
stage_owners_lock();
16531681
owner->next = g_stage_owners;
16541682
g_stage_owners = owner;
@@ -2641,22 +2669,79 @@ static char *create_staging_path(const char *final_path) {
26412669
}
26422670
memcpy(path, final_path, final_len);
26432671
memcpy(path + final_len, suffix, sizeof(suffix));
2644-
int fd = cbm_mkstemp(path);
2645-
if (fd < 0) {
2646-
free(path);
2647-
return NULL;
2648-
}
2649-
#ifdef _WIN32
2650-
_close(fd);
2651-
#else
2652-
close(fd);
2672+
/* The six random chars sit directly after the ".stage." marker; each
2673+
* attempt overwrites the "XXXXXX" template in place. Alphanumerics only,
2674+
* matching stage_suffix_at()/stage_entry_stage_length() so the sweep
2675+
* recognises the minted name and its sidecars. */
2676+
char *random_at = path + final_len + (sizeof(cbm_stage_marker) - 1);
2677+
static const char alphabet[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
2678+
2679+
/* Lock BEFORE the stage becomes visible: take the sidecar lock first, then
2680+
* create the stage's main file with O_EXCL. The main file is therefore
2681+
* never present on disk without its lock already held, so a concurrent
2682+
* run's sweep_orphan_stages() against the same final_path can only ever
2683+
* find this stage lock-held -- it reaches the stage through the .lock
2684+
* sidecar too (stage_entry_stage_length() matches it), probes the lock,
2685+
* sees a live holder, and keeps it. That closes the old create->register
2686+
* window that let a racing sweep delete a live-but-unlocked stage (POSIX)
2687+
* or collide on the sidecar with EACCES (Windows) -- #2111's windows-guards
2688+
* red. A pre-lock-era orphan minted by an OLDER binary still carries no
2689+
* lock, so the sweep's ENOENT path still removes it (#1839 preserved).
2690+
*
2691+
* A minted suffix collides with an existing stage only about 1 in 62^6;
2692+
* retry a bounded number of times, the way mkstemp/mkdtemp do, then fail. */
2693+
for (int attempt = 0; attempt < 128; attempt++) {
2694+
unsigned char rnd[CBM_STAGE_SUFFIX_RANDOM_CHARS];
2695+
if (!cbm_secure_random(rnd, sizeof(rnd))) {
2696+
free(path);
2697+
errno = EIO;
2698+
return NULL;
2699+
}
2700+
for (size_t i = 0; i < sizeof(rnd); i++) {
2701+
random_at[i] = alphabet[rnd[i] % (sizeof(alphabet) - 1)];
2702+
}
2703+
errno = 0;
2704+
int lock_fd = cbm_pipeline_stage_lock_hold(path);
2705+
if (lock_fd < 0) {
2706+
/* A live twin already owns this exact suffix's sidecar (EAGAIN /
2707+
* EACCES), or the sidecar could not be created. Mint a fresh suffix
2708+
* and try again rather than contend for this one. */
2709+
continue;
2710+
}
2711+
FILE *main_file = cbm_fopen(path, "wbx");
2712+
if (!main_file) {
2713+
/* The suffix collided with a lock-less orphan's main file -- its
2714+
* sidecar was takeable, so it is not a live writer. Never inherit a
2715+
* stranger's bytes: drop the lock, remove the sidecar we just took,
2716+
* and mint a fresh suffix. The orphan's main file is left for a
2717+
* later sweep, which removes it as a pre-lock-era orphan. */
2718+
cbm_pipeline_stage_lock_drop(path, lock_fd);
2719+
continue;
2720+
}
2721+
(void)fclose(main_file);
2722+
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
2723+
/* Main file now exists and its lock is already held. Under the OLD
2724+
* create-then-lock ordering this was the unlocked window; the
2725+
* concurrent-sweep test fires here to prove the stage now survives a
2726+
* racing sweep, and to bind RED if that ordering ever regresses. */
2727+
cbm_pipeline_persist_test_run_after_stage_created();
26532728
#endif
2654-
if (!stage_owner_register(path)) {
2655-
(void)cbm_unlink(path);
2656-
free(path);
2657-
return NULL;
2658-
}
2659-
return path;
2729+
if (!stage_owner_adopt(path, lock_fd)) {
2730+
cbm_pipeline_stage_lock_drop(path, lock_fd);
2731+
(void)cbm_unlink(path);
2732+
free(path);
2733+
return NULL;
2734+
}
2735+
return path;
2736+
}
2737+
/* Every attempt failed to take a lock -- keep the observability the old
2738+
* stage_owner_register() emitted for a lock failure. */
2739+
char errno_text[16];
2740+
(void)snprintf(errno_text, sizeof(errno_text), "%d", errno);
2741+
cbm_log_warn("pipeline.stage", "action", "lock_failed", "errno", errno_text, "path", path);
2742+
free(path);
2743+
errno = EEXIST;
2744+
return NULL;
26602745
}
26612746

26622747
/* A backup-failed destination may still have the only recoverable WAL or
@@ -2876,8 +2961,18 @@ static void sweep_one_stage(const char *stage_path) {
28762961
"errno", errno_text, "path", stage_path);
28772962
return;
28782963
}
2879-
/* No owner: absent sidecar (pre-ownership stage, or dropped) or a lock
2880-
* the kernel released with its writer. Ours now, from the lock down. */
2964+
/* No owner: absent sidecar (an ENOENT probe) or a lock the kernel released
2965+
* with its writer. Ours now, from the lock down.
2966+
*
2967+
* The absent-sidecar (ENOENT) case is exactly a pre-lock-era orphan: a
2968+
* stage an OLDER binary minted with no sidecar at all (#1839 pins that
2969+
* these ARE swept). It is NOT an in-flight stage of a current run: since
2970+
* create_staging_path() now takes the sidecar lock BEFORE the stage's main
2971+
* file becomes visible on disk, a live stage always has its sidecar, so a
2972+
* concurrent sweep landing here for one would instead find the lock held
2973+
* above and keep it. Removing on ENOENT therefore reclaims genuine orphans
2974+
* without ever deleting a live stage (the create->register race behind
2975+
* #2111's windows-guards red is closed at the source). */
28812976
int64_t bytes = stage_bytes_on_disk(stage_path);
28822977
remove_stage_files(stage_path);
28832978
if (lock_fd >= 0) {

‎src/pipeline/pipeline_internal.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -908,6 +908,15 @@ void cbm_pipeline_incremental_test_fail_adr_capture_once(void);
908908
typedef void (*cbm_pipeline_test_hook_fn)(void *userdata);
909909
void cbm_pipeline_incremental_test_before_final_manifest_once(cbm_pipeline_test_hook_fn hook,
910910
void *userdata);
911+
/* Fires from create_staging_path(), right after the stage's main file is
912+
* created with O_EXCL. In the current lock-before-visible ordering its sidecar
913+
* lock is already held at this point, so a test hook installed here can run a
914+
* concurrent sweep (via another cbm_pipeline_run() against the same
915+
* final_path) and confirm the just-created stage survives it. Under the OLD
916+
* create-then-lock ordering this was the unlocked window, so the hook also
917+
* binds RED if that ordering regresses. */
918+
void cbm_pipeline_incremental_test_after_stage_created_once(cbm_pipeline_test_hook_fn hook,
919+
void *userdata);
911920
cbm_incremental_route_t cbm_pipeline_incremental_test_last_route(void);
912921
void cbm_pipeline_incremental_test_reset_faults(void);
913922

@@ -917,6 +926,7 @@ bool cbm_pipeline_persist_test_take_failure_after_stage_dump(void);
917926
bool cbm_pipeline_persist_test_take_cancel_after_predump(void);
918927
bool cbm_pipeline_persist_test_take_cancel_after_destination_prepare(void);
919928
void cbm_pipeline_persist_test_run_before_final_manifest(void);
929+
void cbm_pipeline_persist_test_run_after_stage_created(void);
920930
void cbm_pipeline_persist_test_reset_faults(void);
921931
#endif
922932

‎tests/test_pipeline.c‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3707,6 +3707,130 @@ TEST(pipeline_sweep_removes_dead_writer_stage_keeps_locked_stage) {
37073707
PASS();
37083708
}
37093709

3710+
/* #2111 create->register TOCTOU guard. The bug this pins: create_staging_path()
3711+
* used to make the stage's main file visible (via mkstemp) BEFORE taking its
3712+
* sidecar lock, and sweep_orphan_stages() treats an absent lock sidecar
3713+
* (ENOENT) as a confirmed-dead writer (kernel released the lock on death). So a
3714+
* second, concurrent cbm_pipeline_run() against the SAME final_path, landing
3715+
* its own sweep in that narrow unlocked window, removed the first run's
3716+
* in-flight stage out from under it: every extraction pass still completed
3717+
* (none touch the stage file on disk), but the publish that followed found its
3718+
* own stage gone. Two writers racing the same project is a real scenario this
3719+
* PR's own sweep exists to clean up after (auto_index; the recently-fixed
3720+
* stale-rendezvous-recovery retry path) -- not hypothetical, and exactly the
3721+
* shape of #2111's windows-guards red (every pass logged success, nothing was
3722+
* ever committed, "Pipeline failed" surfaced generic; on Windows the sidecar
3723+
* collision surfaced as EACCES/errno=13).
3724+
*
3725+
* The fix takes the sidecar lock BEFORE the main file becomes visible, so this
3726+
* hook -- fired the instant the main file exists -- finds the stage already
3727+
* lock-protected and the racing sweep keeps it. The hook fires at the same
3728+
* point under the old ordering, where the lock was NOT yet held, so this test
3729+
* goes RED if that ordering ever regresses.
3730+
*
3731+
* The racing run is cancelled immediately so it never reaches ITS OWN
3732+
* publish -- isolating the sweep's effect on the first run's stage from the
3733+
* separate question of two full runs both completing for the same project. */
3734+
typedef struct {
3735+
const char *tmp_dir;
3736+
const char *db_path;
3737+
bool stage_survived;
3738+
} racing_sweep_arg_t;
3739+
3740+
static bool find_sole_stage_path(const char *dir, const char *db_basename, char *out,
3741+
size_t out_sz) {
3742+
cbm_dir_t *d = cbm_opendir(dir);
3743+
if (!d) {
3744+
return false;
3745+
}
3746+
char prefix[256];
3747+
snprintf(prefix, sizeof(prefix), "%s.stage.", db_basename);
3748+
size_t prefix_len = strlen(prefix);
3749+
bool found = false;
3750+
cbm_dirent_t *entry;
3751+
while ((entry = cbm_readdir(d)) != NULL) {
3752+
size_t name_len = strlen(entry->name);
3753+
enum { STAGE_SUFFIX_RANDOM_CHARS = 6 }; /* mirrors CBM_STAGE_SUFFIX_RANDOM_CHARS */
3754+
if (strncmp(entry->name, prefix, prefix_len) == 0 &&
3755+
name_len == prefix_len + STAGE_SUFFIX_RANDOM_CHARS) {
3756+
snprintf(out, out_sz, "%s/%s", dir, entry->name);
3757+
found = true;
3758+
break;
3759+
}
3760+
}
3761+
cbm_closedir(d);
3762+
return found;
3763+
}
3764+
3765+
static void *racing_sweep_thread(void *arg) {
3766+
racing_sweep_arg_t *a = (racing_sweep_arg_t *)arg;
3767+
cbm_pipeline_t *p = cbm_pipeline_new(a->tmp_dir, a->db_path, CBM_MODE_FULL);
3768+
if (p) {
3769+
/* Its own sweep_orphan_stages() runs at the very start, before this
3770+
* run mints its own stage -- exactly like the first run's. Cancel
3771+
* immediately: this run must reach the sweep and nothing past it. */
3772+
cbm_pipeline_cancel(p);
3773+
(void)cbm_pipeline_run(p);
3774+
cbm_pipeline_free(p);
3775+
}
3776+
return NULL;
3777+
}
3778+
3779+
/* Fired from inside the FIRST run's create_staging_path(), the instant its
3780+
* stage main file exists (under the fix, with its sidecar lock already held;
3781+
* under the old create-then-lock ordering, before the lock was taken): run a
3782+
* second, cancelled cbm_pipeline_run() for the same project synchronously on
3783+
* another thread, so its sweep has every chance to reach the stage before
3784+
* control returns to the first run. */
3785+
static void racing_sweep_hook(void *userdata) {
3786+
racing_sweep_arg_t *a = (racing_sweep_arg_t *)userdata;
3787+
char stage_path[600] = {0};
3788+
bool had_stage =
3789+
find_sole_stage_path(a->tmp_dir, "generation.db", stage_path, sizeof(stage_path));
3790+
cbm_thread_t tid;
3791+
if (cbm_thread_create(&tid, 0, racing_sweep_thread, a) == 0) {
3792+
cbm_thread_join(&tid);
3793+
}
3794+
a->stage_survived = had_stage && path_exists(stage_path);
3795+
}
3796+
3797+
TEST(pipeline_concurrent_sweep_must_not_remove_inflight_stage) {
3798+
char tmp[256];
3799+
snprintf(tmp, sizeof(tmp), "/tmp/cbm_stage_race_XXXXXX");
3800+
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
3801+
write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 1\n");
3802+
char db_path[512];
3803+
snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp);
3804+
3805+
cbm_pipeline_incremental_test_reset_faults();
3806+
racing_sweep_arg_t race_arg = {.tmp_dir = tmp, .db_path = db_path, .stage_survived = false};
3807+
cbm_pipeline_incremental_test_after_stage_created_once(racing_sweep_hook, &race_arg);
3808+
3809+
cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL);
3810+
ASSERT_NOT_NULL(p);
3811+
char project[256];
3812+
snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p));
3813+
int rc = cbm_pipeline_run(p);
3814+
cbm_pipeline_free(p);
3815+
3816+
bool db_exists = path_exists(db_path);
3817+
int defined_count = -1;
3818+
int absent_count = -1;
3819+
observe_named_generation(db_path, project, "StableGeneration", "NeverDefined", &defined_count,
3820+
&absent_count);
3821+
int stage_count = count_generation_stage_artifacts(tmp, "generation.db");
3822+
cbm_pipeline_incremental_test_reset_faults();
3823+
th_rmtree(tmp);
3824+
3825+
ASSERT_TRUE(race_arg.stage_survived);
3826+
ASSERT_EQ(rc, 0);
3827+
ASSERT_TRUE(db_exists);
3828+
ASSERT_EQ(defined_count, 1);
3829+
ASSERT_EQ(absent_count, 0);
3830+
ASSERT_EQ(stage_count, 0);
3831+
PASS();
3832+
}
3833+
37103834
static char g_route_log_capture[8192];
37113835
static atomic_flag g_route_log_spin = ATOMIC_FLAG_INIT;
37123836

@@ -14473,6 +14597,7 @@ SUITE(pipeline_semantic_manifest_repro) {
1447314597
RUN_TEST(pipeline_minted_stage_is_owned_until_released);
1447414598
RUN_TEST(pipeline_stale_zero_byte_stage_beside_valid_db_routes_incremental_and_is_swept);
1447514599
RUN_TEST(pipeline_sweep_removes_dead_writer_stage_keeps_locked_stage);
14600+
RUN_TEST(pipeline_concurrent_sweep_must_not_remove_inflight_stage);
1447614601
RUN_TEST(pipeline_fresh_index_never_reports_invalid_existing_db);
1447714602
RUN_TEST(pipeline_source_mutation_before_publication_preserves_previous_generation);
1447814603
RUN_TEST(pipeline_source_addition_before_publication_preserves_previous_generation);

0 commit comments

Comments
 (0)