Skip to content

Commit b6458f9

Browse files
DeusDataclaude
andcommitted
fix(pipeline): sweep the orphaned stages of a dead writer before each run
Nothing on the worker-death path ever cleaned a stage up: the supervisor reaps only an exit status, and a cancel is SIGTERM then SIGKILL after one second, which a gigabyte backup or clone never finishes inside. Every daemon-side cancel or OOM kill therefore left a full-size <db>.stage.X (plus -wal/-shm) behind for good, and with auto_index a large project filled the disk with copies of itself (DeusData#1839). Each run now sweeps before minting its own stage. Only names of the exact minted shape for THIS database are considered -- "<basename>.stage.<6 alphanumerics>" and that stage's -wal/-shm/-journal/.lock sidecars -- never the live database, a quarantined .corrupt, or another project's files. Liveness is the ownership lock and nothing else: a stage whose sidecar lock can be taken has no writer and is removed (logged as pipeline.stage action=orphan_removed bytes=N); one whose lock a live writer holds is kept (orphan_kept reason=live_writer). No pid, no mtime, no grace period, so the sweep is deterministic and never races a writer. A stage minted by an older binary carries no lock and is swept too; that writer's final rename then fails and it discards its stage. The live database is never named on either path. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 0c4831a commit b6458f9

2 files changed

Lines changed: 345 additions & 0 deletions

File tree

‎src/pipeline/pipeline.c‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2741,6 +2741,190 @@ static int export_after_publish(cbm_pipeline_t *p, const char *final_path) {
27412741
return 0;
27422742
}
27432743

2744+
/* ── Orphan sweep (#1839) ────────────────────────────────────────
2745+
*
2746+
* Nothing on the worker-death path ever cleaned a stage up, so the sweep
2747+
* runs at the start of every run, before this run mints its own stage. It
2748+
* considers ONLY names of the exact minted shape for THIS database --
2749+
* "<basename>.stage.<6 alphanumerics>" plus that stage's -wal/-shm/-journal
2750+
* and .lock sidecars -- never the live database, a quarantined .corrupt, or
2751+
* another project's files. A stage is removed when its ownership lock can be
2752+
* taken (its writer is dead, or the stage predates ownership) and kept when
2753+
* a live writer holds the lock. The pre-ownership case is the one honest
2754+
* gap: a stage an OLDER binary is still writing against this database has
2755+
* no lock and is swept; that writer's final rename then fails and it
2756+
* discards. The live database is never named here on either path. */
2757+
2758+
static const char *const cbm_stage_sidecar_tails[] = {"", "-wal", "-shm", "-journal", ".lock"};
2759+
2760+
/* If `name` is "<base>.stage.<6 alphanumerics><known tail>", return the
2761+
* length of the stage name proper (without the tail); 0 otherwise. */
2762+
static size_t stage_entry_stage_length(const char *name, const char *base, size_t base_len) {
2763+
if (strncmp(name, base, base_len) != 0) {
2764+
return 0;
2765+
}
2766+
const char *at = name + base_len;
2767+
if (strncmp(at, cbm_stage_marker, sizeof(cbm_stage_marker) - 1) != 0) {
2768+
return 0;
2769+
}
2770+
at += sizeof(cbm_stage_marker) - 1;
2771+
for (int i = 0; i < CBM_STAGE_SUFFIX_RANDOM_CHARS; i++) {
2772+
/* NUL is not alphanumeric, so a short name fails here too. */
2773+
if (!isalnum((unsigned char)at[i])) {
2774+
return 0;
2775+
}
2776+
}
2777+
at += CBM_STAGE_SUFFIX_RANDOM_CHARS;
2778+
for (size_t i = 0; i < sizeof(cbm_stage_sidecar_tails) / sizeof(cbm_stage_sidecar_tails[0]);
2779+
i++) {
2780+
if (strcmp(at, cbm_stage_sidecar_tails[i]) == 0) {
2781+
return (size_t)(at - name);
2782+
}
2783+
}
2784+
return 0;
2785+
}
2786+
2787+
typedef struct {
2788+
char **names;
2789+
int count;
2790+
int cap;
2791+
} stage_name_list_t;
2792+
2793+
/* Add a stage name once, however many of its files were listed. */
2794+
static void stage_name_list_add(stage_name_list_t *list, const char *name, size_t len) {
2795+
for (int i = 0; i < list->count; i++) {
2796+
if (strlen(list->names[i]) == len && strncmp(list->names[i], name, len) == 0) {
2797+
return;
2798+
}
2799+
}
2800+
if (list->count == list->cap) {
2801+
int cap = list->cap ? list->cap * 2 : 8;
2802+
char **grown = (char **)realloc(list->names, (size_t)cap * sizeof(*grown));
2803+
if (!grown) {
2804+
return;
2805+
}
2806+
list->names = grown;
2807+
list->cap = cap;
2808+
}
2809+
char *copy = (char *)malloc(len + 1);
2810+
if (!copy) {
2811+
return;
2812+
}
2813+
memcpy(copy, name, len);
2814+
copy[len] = '\0';
2815+
list->names[list->count++] = copy;
2816+
}
2817+
2818+
static int64_t stage_bytes_on_disk(const char *stage_path) {
2819+
static const char *const files[] = {"", "-wal", "-shm", "-journal"};
2820+
int64_t total = 0;
2821+
char side[CBM_SZ_4K];
2822+
for (size_t i = 0; i < sizeof(files) / sizeof(files[0]); i++) {
2823+
int n = snprintf(side, sizeof(side), "%s%s", stage_path, files[i]);
2824+
if (n <= 0 || (size_t)n >= sizeof(side)) {
2825+
continue;
2826+
}
2827+
cbm_path_info_t info;
2828+
if (cbm_path_info_utf8(side, &info) == CBM_PATH_INFO_OK && info.is_regular) {
2829+
total += info.size;
2830+
}
2831+
}
2832+
return total;
2833+
}
2834+
2835+
static void sweep_one_stage(const char *stage_path) {
2836+
char *sidecar = stage_lock_sidecar_path(stage_path);
2837+
if (!sidecar) {
2838+
return;
2839+
}
2840+
errno = 0;
2841+
int lock_fd = cbm_lockfile_open(sidecar, false);
2842+
int probe_errno = errno;
2843+
free(sidecar);
2844+
if (lock_fd < 0 && probe_errno != ENOENT) {
2845+
bool live = probe_errno == EAGAIN || probe_errno == EACCES;
2846+
#if defined(EWOULDBLOCK) && EWOULDBLOCK != EAGAIN
2847+
live = live || probe_errno == EWOULDBLOCK;
2848+
#endif
2849+
if (live) {
2850+
cbm_log_info("pipeline.stage", "action", "orphan_kept", "reason", "live_writer", "path",
2851+
stage_path);
2852+
return;
2853+
}
2854+
char errno_text[16];
2855+
(void)snprintf(errno_text, sizeof(errno_text), "%d", probe_errno);
2856+
cbm_log_warn("pipeline.stage", "action", "orphan_kept", "reason", "lock_probe_failed",
2857+
"errno", errno_text, "path", stage_path);
2858+
return;
2859+
}
2860+
/* No owner: absent sidecar (pre-ownership stage, or dropped) or a lock
2861+
* the kernel released with its writer. Ours now, from the lock down. */
2862+
int64_t bytes = stage_bytes_on_disk(stage_path);
2863+
remove_stage_files(stage_path);
2864+
if (lock_fd >= 0) {
2865+
cbm_pipeline_stage_lock_drop(stage_path, lock_fd);
2866+
}
2867+
char bytes_text[32];
2868+
(void)snprintf(bytes_text, sizeof(bytes_text), "%lld", (long long)bytes);
2869+
cbm_log_info("pipeline.stage", "action", "orphan_removed", "bytes", bytes_text, "path",
2870+
stage_path);
2871+
}
2872+
2873+
static void sweep_orphan_stages(const char *final_path) {
2874+
/* Directory part INCLUDING its trailing separator, so the stage paths
2875+
* are joined exactly as the final path was spelled. */
2876+
size_t prefix_len = 0;
2877+
for (const char *c = final_path; *c; c++) {
2878+
if (*c == '/'
2879+
#ifdef _WIN32
2880+
|| *c == '\\'
2881+
#endif
2882+
) {
2883+
prefix_len = (size_t)(c - final_path) + 1;
2884+
}
2885+
}
2886+
const char *base = final_path + prefix_len;
2887+
size_t base_len = strlen(base);
2888+
if (base_len == 0) {
2889+
return;
2890+
}
2891+
char *dir_path = prefix_len ? (char *)malloc(prefix_len + 1) : strdup(".");
2892+
if (!dir_path) {
2893+
return;
2894+
}
2895+
if (prefix_len) {
2896+
memcpy(dir_path, final_path, prefix_len);
2897+
dir_path[prefix_len] = '\0';
2898+
}
2899+
cbm_dir_t *dir = cbm_opendir(dir_path);
2900+
if (!dir) {
2901+
free(dir_path);
2902+
return;
2903+
}
2904+
stage_name_list_t list = {0};
2905+
cbm_dirent_t *entry;
2906+
while ((entry = cbm_readdir(dir)) != NULL) {
2907+
size_t stage_len = stage_entry_stage_length(entry->name, base, base_len);
2908+
if (stage_len) {
2909+
stage_name_list_add(&list, entry->name, stage_len);
2910+
}
2911+
}
2912+
cbm_closedir(dir);
2913+
for (int i = 0; i < list.count; i++) {
2914+
size_t name_len = strlen(list.names[i]);
2915+
char *stage_path = (char *)malloc(prefix_len + name_len + 1);
2916+
if (stage_path) {
2917+
memcpy(stage_path, final_path, prefix_len);
2918+
memcpy(stage_path + prefix_len, list.names[i], name_len + 1);
2919+
sweep_one_stage(stage_path);
2920+
free(stage_path);
2921+
}
2922+
free(list.names[i]);
2923+
}
2924+
free(list.names);
2925+
free(dir_path);
2926+
}
2927+
27442928
int cbm_pipeline_run(cbm_pipeline_t *p) {
27452929
if (!p) {
27462930
return CBM_NOT_FOUND;
@@ -2752,6 +2936,7 @@ int cbm_pipeline_run(cbm_pipeline_t *p) {
27522936
}
27532937
struct stat final_st;
27542938
bool final_existed = stat(final_path, &final_st) == 0;
2939+
sweep_orphan_stages(final_path);
27552940
char *staging_path = create_staging_path(final_path);
27562941
if (!staging_path) {
27572942
free(final_path);

‎tests/test_pipeline.c‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3549,6 +3549,164 @@ TEST(pipeline_minted_stage_is_owned_until_released) {
35493549
PASS();
35503550
}
35513551

3552+
static bool file_has_content(const char *path, const char *expected) {
3553+
FILE *f = cbm_fopen(path, "rb");
3554+
if (!f) {
3555+
return false;
3556+
}
3557+
char buf[256] = {0};
3558+
size_t n = fread(buf, 1, sizeof(buf) - 1, f);
3559+
(void)fclose(f);
3560+
return n == strlen(expected) && memcmp(buf, expected, n) == 0;
3561+
}
3562+
3563+
/* #1839 / #1864: a stage a dead writer left beside a VALID database is swept
3564+
* by the next run and never influences its route. The 0-byte shape is what a
3565+
* worker killed before its backup wrote a page leaves behind; a stale -shm
3566+
* beside it is swept with it. */
3567+
TEST(pipeline_stale_zero_byte_stage_beside_valid_db_routes_incremental_and_is_swept) {
3568+
char tmp[256];
3569+
snprintf(tmp, sizeof(tmp), "/tmp/cbm_stale_stage_XXXXXX");
3570+
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
3571+
write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 1\n");
3572+
char db_path[512];
3573+
snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp);
3574+
3575+
cbm_pipeline_incremental_test_reset_faults();
3576+
cbm_pipeline_t *baseline = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL);
3577+
ASSERT_NOT_NULL(baseline);
3578+
ASSERT_EQ(cbm_pipeline_run(baseline), 0);
3579+
char project[256];
3580+
snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(baseline));
3581+
cbm_pipeline_free(baseline);
3582+
3583+
char stale_stage[600];
3584+
char stale_shm[600];
3585+
snprintf(stale_stage, sizeof(stale_stage), "%s.stage.deadbe", db_path);
3586+
snprintf(stale_shm, sizeof(stale_shm), "%s.stage.deadbe-shm", db_path);
3587+
ASSERT_EQ(th_write_file(stale_stage, ""), 0);
3588+
ASSERT_EQ(th_write_file(stale_shm, "stale-shm"), 0);
3589+
3590+
/* A body-only change: no added names, so the planner may repair. */
3591+
write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 2\n");
3592+
cbm_pipeline_incremental_test_reset_faults();
3593+
cbm_pipeline_t *incr = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL);
3594+
ASSERT_NOT_NULL(incr);
3595+
int incr_rc = cbm_pipeline_run(incr);
3596+
cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route();
3597+
cbm_pipeline_free(incr);
3598+
3599+
bool stale_stage_gone = !path_exists(stale_stage);
3600+
bool stale_shm_gone = !path_exists(stale_shm);
3601+
int stage_count = count_generation_stage_artifacts(tmp, "generation.db");
3602+
int stable_count = -1;
3603+
int absent_count = -1;
3604+
observe_named_generation(db_path, project, "StableGeneration", "NeverDefined", &stable_count,
3605+
&absent_count);
3606+
cbm_pipeline_incremental_test_reset_faults();
3607+
th_rmtree(tmp);
3608+
3609+
ASSERT_EQ(incr_rc, 0);
3610+
ASSERT_TRUE(route != CBM_INCREMENTAL_ROUTE_FORCED_FULL);
3611+
ASSERT_TRUE(route != CBM_INCREMENTAL_ROUTE_NONE);
3612+
ASSERT_TRUE(stale_stage_gone);
3613+
ASSERT_TRUE(stale_shm_gone);
3614+
ASSERT_EQ(stage_count, 0);
3615+
ASSERT_EQ(stable_count, 1);
3616+
ASSERT_EQ(absent_count, 0);
3617+
PASS();
3618+
}
3619+
3620+
/* #1839: the sweep removes exactly the stages nobody owns. A dead writer's
3621+
* stage (main, -wal, and the unlocked .lock sidecar its death left) goes; a
3622+
* stage whose writer is LIVE -- here the test, holding its lock -- is kept
3623+
* byte for byte, and goes only once that lock is dropped. No timing: liveness
3624+
* is the kernel lock, nothing else. */
3625+
TEST(pipeline_sweep_removes_dead_writer_stage_keeps_locked_stage) {
3626+
char tmp[256];
3627+
snprintf(tmp, sizeof(tmp), "/tmp/cbm_stage_sweep_XXXXXX");
3628+
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
3629+
write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 1\n");
3630+
char db_path[512];
3631+
snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp);
3632+
3633+
cbm_pipeline_incremental_test_reset_faults();
3634+
cbm_pipeline_t *baseline = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL);
3635+
ASSERT_NOT_NULL(baseline);
3636+
ASSERT_EQ(cbm_pipeline_run(baseline), 0);
3637+
char project[256];
3638+
snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(baseline));
3639+
cbm_pipeline_free(baseline);
3640+
3641+
char dead_stage[600];
3642+
char dead_wal[600];
3643+
char dead_lock[600];
3644+
char live_stage[600];
3645+
char live_lock[600];
3646+
snprintf(dead_stage, sizeof(dead_stage), "%s.stage.aaaaaa", db_path);
3647+
snprintf(dead_wal, sizeof(dead_wal), "%s.stage.aaaaaa-wal", db_path);
3648+
snprintf(dead_lock, sizeof(dead_lock), "%s.stage.aaaaaa.lock", db_path);
3649+
snprintf(live_stage, sizeof(live_stage), "%s.stage.bbbbbb", db_path);
3650+
snprintf(live_lock, sizeof(live_lock), "%s.stage.bbbbbb.lock", db_path);
3651+
static const char live_bytes[] = "live-stage-bytes";
3652+
ASSERT_EQ(th_write_file(dead_stage, "dead-stage-bytes"), 0);
3653+
ASSERT_EQ(th_write_file(dead_wal, "dead-wal"), 0);
3654+
ASSERT_EQ(th_write_file(dead_lock, ""), 0);
3655+
ASSERT_EQ(th_write_file(live_stage, live_bytes), 0);
3656+
int live_fd = cbm_pipeline_stage_lock_hold(live_stage);
3657+
ASSERT_TRUE(live_fd >= 0);
3658+
3659+
write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 2\n");
3660+
cbm_pipeline_incremental_test_reset_faults();
3661+
cbm_pipeline_t *first = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL);
3662+
ASSERT_NOT_NULL(first);
3663+
int first_rc = cbm_pipeline_run(first);
3664+
cbm_pipeline_free(first);
3665+
3666+
bool dead_stage_gone = !path_exists(dead_stage);
3667+
bool dead_wal_gone = !path_exists(dead_wal);
3668+
bool dead_lock_gone = !path_exists(dead_lock);
3669+
bool live_kept = file_has_content(live_stage, live_bytes);
3670+
bool live_lock_kept = path_exists(live_lock);
3671+
int stable_after_first = -1;
3672+
int absent_after_first = -1;
3673+
observe_named_generation(db_path, project, "StableGeneration", "NeverDefined",
3674+
&stable_after_first, &absent_after_first);
3675+
3676+
cbm_pipeline_stage_lock_drop(live_stage, live_fd);
3677+
cbm_pipeline_incremental_test_reset_faults();
3678+
cbm_pipeline_t *second = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL);
3679+
ASSERT_NOT_NULL(second);
3680+
int second_rc = cbm_pipeline_run(second);
3681+
cbm_pipeline_free(second);
3682+
3683+
bool live_gone = !path_exists(live_stage);
3684+
bool live_lock_gone = !path_exists(live_lock);
3685+
int stage_count = count_generation_stage_artifacts(tmp, "generation.db");
3686+
int stable_after_second = -1;
3687+
int absent_after_second = -1;
3688+
observe_named_generation(db_path, project, "StableGeneration", "NeverDefined",
3689+
&stable_after_second, &absent_after_second);
3690+
cbm_pipeline_incremental_test_reset_faults();
3691+
th_rmtree(tmp);
3692+
3693+
ASSERT_EQ(first_rc, 0);
3694+
ASSERT_TRUE(dead_stage_gone);
3695+
ASSERT_TRUE(dead_wal_gone);
3696+
ASSERT_TRUE(dead_lock_gone);
3697+
ASSERT_TRUE(live_kept);
3698+
ASSERT_TRUE(live_lock_kept);
3699+
ASSERT_EQ(stable_after_first, 1);
3700+
ASSERT_EQ(absent_after_first, 0);
3701+
ASSERT_EQ(second_rc, 0);
3702+
ASSERT_TRUE(live_gone);
3703+
ASSERT_TRUE(live_lock_gone);
3704+
ASSERT_EQ(stage_count, 0);
3705+
ASSERT_EQ(stable_after_second, 1);
3706+
ASSERT_EQ(absent_after_second, 0);
3707+
PASS();
3708+
}
3709+
35523710
/* Discovery and extraction must describe the same immutable generation. A
35533711
* source file created after extraction is not present in the original file
35543712
* list, so merely re-hashing that list cannot detect the race. Publication
@@ -14253,6 +14411,8 @@ SUITE(pipeline_semantic_manifest_repro) {
1425314411
RUN_TEST(pipeline_publication_never_uses_a_predictable_staging_path);
1425414412
RUN_TEST(pipeline_stage_names_never_nest);
1425514413
RUN_TEST(pipeline_minted_stage_is_owned_until_released);
14414+
RUN_TEST(pipeline_stale_zero_byte_stage_beside_valid_db_routes_incremental_and_is_swept);
14415+
RUN_TEST(pipeline_sweep_removes_dead_writer_stage_keeps_locked_stage);
1425614416
RUN_TEST(pipeline_source_mutation_before_publication_preserves_previous_generation);
1425714417
RUN_TEST(pipeline_source_addition_before_publication_preserves_previous_generation);
1425814418
RUN_TEST(pipeline_tsconfig_mutation_before_publication_preserves_previous_generation);

0 commit comments

Comments
 (0)