Skip to content

Commit 1a38d0a

Browse files
authored
Merge pull request DeusData#1016 from DeusData/fix/stale-wal-sidecars
fix(store): remove destination WAL/SHM sidecars before installing a fresh DB
2 parents d7e6082 + 43d84ee commit 1a38d0a

6 files changed

Lines changed: 178 additions & 4 deletions

File tree

‎internal/cbm/sqlite_writer.c‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
#include "sqlite_writer.h"
1818
#include "foundation/constants.h"
19+
#include "foundation/compat_fs.h"
1920
#include "foundation/compat_thread.h"
2021
#include "foundation/profile.h"
2122

@@ -2206,6 +2207,10 @@ struct cbm_db_writer {
22062207
};
22072208

22082209
cbm_db_writer_t *cbm_writer_open(const char *path) {
2210+
/* Installing a fresh DB generation: drop the destination's leftover
2211+
* -wal/-shm or a crashed session's WAL gets replayed on top of the
2212+
* new file at the next open (#897). */
2213+
cbm_remove_db_sidecars(path);
22092214
FILE *fp = fopen(path, "wb");
22102215
if (!fp) {
22112216
return NULL;

‎src/foundation/compat_fs.c‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -713,3 +713,47 @@ int cbm_exec_no_shell(const char *const *argv) {
713713
}
714714

715715
#endif /* _WIN32 */
716+
717+
/* rename() with overwrite semantics on every platform: POSIX rename already
718+
* replaces atomically; Windows rename fails with EEXIST when the target
719+
* exists, so use MoveFileExW(MOVEFILE_REPLACE_EXISTING) there (wide paths —
720+
* raw MoveFileExA would re-mangle non-ASCII cache paths). */
721+
int cbm_rename_replace(const char *src, const char *dst) {
722+
#ifdef _WIN32
723+
wchar_t *wsrc = cbm_utf8_to_wide(src);
724+
wchar_t *wdst = cbm_utf8_to_wide(dst);
725+
int ret = CBM_NOT_FOUND;
726+
if (wsrc && wdst) {
727+
ret = MoveFileExW(wsrc, wdst, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED)
728+
? 0
729+
: CBM_NOT_FOUND;
730+
}
731+
free(wsrc);
732+
free(wdst);
733+
return ret;
734+
#else
735+
return rename(src, dst);
736+
#endif
737+
}
738+
739+
/* Remove a SQLite database's -wal/-shm sidecars (both platforms). Any code
740+
* path that installs a FRESH database file at a path where a previous
741+
* generation lived must call this first: SQLite decides whether to replay a
742+
* WAL purely from the sidecar's own header/checksums, so a leftover WAL
743+
* from a crashed session is recovered ON TOP of the freshly installed file
744+
* at the next open, splicing old-generation pages into it (#897). */
745+
void cbm_remove_db_sidecars(const char *db_path) {
746+
if (!db_path || !db_path[0]) {
747+
return;
748+
}
749+
enum { SIDECAR_PATH_MAX = 4096 };
750+
char side[SIDECAR_PATH_MAX];
751+
int n = snprintf(side, sizeof(side), "%s-wal", db_path);
752+
if (n > 0 && (size_t)n < sizeof(side)) {
753+
(void)cbm_unlink(side);
754+
}
755+
n = snprintf(side, sizeof(side), "%s-shm", db_path);
756+
if (n > 0 && (size_t)n < sizeof(side)) {
757+
(void)cbm_unlink(side);
758+
}
759+
}

‎src/foundation/compat_fs.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,13 @@ bool cbm_mkdir_p(const char *path, int mode);
4646

4747
/* Delete a file. Returns 0 on success. */
4848
int cbm_unlink(const char *path);
49+
/* Remove <db_path>-wal/-shm. MUST be called by any path installing a fresh
50+
* DB file where a previous generation lived — a leftover WAL is otherwise
51+
* replayed on top of the new file at the next open (#897). */
52+
void cbm_remove_db_sidecars(const char *db_path);
53+
/* rename() that replaces an existing destination on every platform
54+
* (Windows rename fails with EEXIST; this uses MoveFileExW there). */
55+
int cbm_rename_replace(const char *src, const char *dst);
4956

5057
/* Delete an empty directory. Returns 0 on success. */
5158
int cbm_rmdir(const char *path);

‎src/pipeline/artifact.c‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -710,7 +710,11 @@ int cbm_artifact_import(const char *repo_path, const char *cache_db_path) {
710710

711711
cbm_store_close(store);
712712

713-
/* Atomic rename to final path */
713+
/* Atomic rename to final path. Drop the DESTINATION's leftover
714+
* -wal/-shm first: the import cleans the tmp file's sidecars, but a
715+
* stale WAL next to the cache path would be replayed on top of the
716+
* imported file at the next open (#897). */
717+
cbm_remove_db_sidecars(cache_db_path);
714718
if (rename(tmp_path, cache_db_path) != 0) {
715719
cbm_log_error("artifact.import", "err", "rename_to_cache");
716720
cbm_unlink(tmp_path);

‎src/store/store.c‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ enum {
6161

6262
#define SLEN(s) (sizeof(s) - 1)
6363
#include "store/store.h"
64+
#include "foundation/compat_fs.h"
6465
#include "foundation/platform.h"
6566
#include "foundation/compat.h"
6667
#include "foundation/log.h"
@@ -1120,9 +1121,12 @@ int cbm_store_dump_to_file(cbm_store_t *s, const char *dest_path) {
11201121
sqlite3_exec(dest_db, "PRAGMA journal_mode = WAL;", NULL, NULL, NULL);
11211122
sqlite3_close(dest_db);
11221123

1123-
/* Atomic rename: old WAL/SHM become stale and get recreated by
1124-
* the next reader's configure_pragmas call. */
1125-
if (rename(tmp_path, dest_path) != 0) {
1124+
/* Remove the DESTINATION's leftover sidecars before installing: a
1125+
* stale WAL from a crashed session would be replayed on top of the
1126+
* fresh file at the next open — SQLite validates the WAL against its
1127+
* own header/checksums, not against the main file (#897). */
1128+
cbm_remove_db_sidecars(dest_path);
1129+
if (cbm_rename_replace(tmp_path, dest_path) != 0) {
11261130
store_set_error(s, "dump: rename failed");
11271131
(void)unlink(tmp_path);
11281132
return CBM_STORE_ERR;

‎tests/test_store_checkpoint.c‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,116 @@ TEST(checkpoint_does_not_truncate_wal) {
7474
PASS();
7575
}
7676

77+
78+
/* #897: any code path installing a fresh DB file must delete the
79+
* destination's -wal/-shm first. SQLite decides whether to replay a WAL
80+
* purely from the sidecar's own header/checksums — a leftover WAL from a
81+
* crashed previous session is recovered ON TOP of the freshly installed
82+
* file at the next open, splicing old-generation pages into it (short
83+
* indexes, btreeInitPage failures, or resurrected stale rows).
84+
*
85+
* Repro (per the issue): hot-copy a live WAL aside, close cleanly, restore
86+
* the copy as the crashed-session leftover, install a fresh generation via
87+
* cbm_store_dump_to_file, reopen — the stale generation's row must NOT be
88+
* visible and the fresh row must be. */
89+
static int tsc_copy_file(const char *src, const char *dst) {
90+
FILE *in = fopen(src, "rb");
91+
if (!in) {
92+
return -1;
93+
}
94+
FILE *out = fopen(dst, "wb");
95+
if (!out) {
96+
(void)fclose(in);
97+
return -1;
98+
}
99+
char buf[4096];
100+
size_t n;
101+
while ((n = fread(buf, 1, sizeof(buf), in)) > 0) {
102+
if (fwrite(buf, 1, n, out) != n) {
103+
(void)fclose(in);
104+
(void)fclose(out);
105+
return -1;
106+
}
107+
}
108+
(void)fclose(in);
109+
(void)fclose(out);
110+
return 0;
111+
}
112+
113+
TEST(dump_install_ignores_stale_wal_sidecar) {
114+
char *td = th_mktempdir("cbm_stalewal");
115+
char db_path[512];
116+
char wal_path[512];
117+
char stale_copy[512];
118+
snprintf(db_path, sizeof(db_path), "%s/gen.db", td);
119+
snprintf(wal_path, sizeof(wal_path), "%s-wal", db_path);
120+
snprintf(stale_copy, sizeof(stale_copy), "%s/stale.wal", td);
121+
122+
/* Generation 1: file-backed store with a marker row living in the WAL. */
123+
cbm_store_t *s1 = cbm_store_open_path(db_path);
124+
ASSERT_NOT_NULL(s1);
125+
cbm_store_upsert_project(s1, "walgen", "/tmp/walgen");
126+
cbm_node_t stale = {.project = "walgen",
127+
.label = "Function",
128+
.name = "stale_gen_node",
129+
.qualified_name = "walgen.mod.stale_gen_node",
130+
.file_path = "mod.py",
131+
.start_line = 1,
132+
.end_line = 2};
133+
ASSERT_TRUE(cbm_store_upsert_node(s1, &stale) > 0);
134+
135+
/* Hot-copy the live WAL (must be non-empty or the repro is vacuous). */
136+
struct stat st_wal = {0};
137+
ASSERT_EQ(stat(wal_path, &st_wal), 0);
138+
ASSERT_TRUE(st_wal.st_size > 0);
139+
ASSERT_EQ(tsc_copy_file(wal_path, stale_copy), 0);
140+
cbm_store_close(s1); /* clean close checkpoints + removes the WAL */
141+
142+
/* Simulate the crashed previous session's leftover sidecar. */
143+
ASSERT_EQ(tsc_copy_file(stale_copy, wal_path), 0);
144+
145+
/* Generation 2: fresh store installed over db_path. */
146+
cbm_store_t *s2 = cbm_store_open_memory();
147+
ASSERT_NOT_NULL(s2);
148+
cbm_store_upsert_project(s2, "walgen", "/tmp/walgen");
149+
cbm_node_t fresh = {.project = "walgen",
150+
.label = "Function",
151+
.name = "fresh_gen_node",
152+
.qualified_name = "walgen.mod.fresh_gen_node",
153+
.file_path = "mod.py",
154+
.start_line = 1,
155+
.end_line = 2};
156+
ASSERT_TRUE(cbm_store_upsert_node(s2, &fresh) > 0);
157+
ASSERT_EQ(cbm_store_dump_to_file(s2, db_path), CBM_STORE_OK);
158+
cbm_store_close(s2);
159+
160+
/* Reader: the stale WAL must not have been replayed onto gen 2. */
161+
cbm_store_t *s3 = cbm_store_open_path(db_path);
162+
ASSERT_NOT_NULL(s3);
163+
cbm_node_t *hits = NULL;
164+
int hit_count = 0;
165+
ASSERT_EQ(cbm_store_find_nodes_by_name(s3, "walgen", "fresh_gen_node", &hits, &hit_count),
166+
CBM_STORE_OK);
167+
ASSERT_TRUE(hit_count >= 1);
168+
cbm_store_free_nodes(hits, hit_count);
169+
hits = NULL;
170+
hit_count = 0;
171+
ASSERT_EQ(cbm_store_find_nodes_by_name(s3, "walgen", "stale_gen_node", &hits, &hit_count),
172+
CBM_STORE_OK);
173+
ASSERT_EQ(hit_count, 0);
174+
cbm_store_free_nodes(hits, hit_count);
175+
cbm_store_close(s3);
176+
177+
unlink(wal_path);
178+
char shm_path[512];
179+
snprintf(shm_path, sizeof(shm_path), "%s-shm", db_path);
180+
unlink(shm_path);
181+
unlink(db_path);
182+
unlink(stale_copy);
183+
PASS();
184+
}
185+
77186
SUITE(store_checkpoint) {
78187
RUN_TEST(checkpoint_does_not_truncate_wal);
188+
RUN_TEST(dump_install_ignores_stale_wal_sidecar);
79189
}

0 commit comments

Comments
 (0)