fix(store): bound the WAL with journal_size_limit so a starved log is reclaimed

On-disk write connections set no journal_size_limit, so the -wal file is never
physically shrunk: all our checkpoints are PASSIVE (TRUNCATE is deliberately
avoided because its ftruncate(fd,0) can SIGBUS a sibling mmap'd process on
macOS), and PASSIVE never ftruncates. Under checkpoint starvation the WAL then
grows without bound (reported: 115 GB, filled the drive) and even a later
successful checkpoint leaves the file at its peak size (DeusData#1083).

Set journal_size_limit = 256 MiB on write connections so SQLite truncates the
-wal back to that bound whenever a checkpoint can finally reset the log —
reclaiming disk opportunistically during a session, not only on clean close.
256 MiB is far above the healthy WAL (~4 MiB under the default 1000-page
autocheckpoint), so normal indexing never triggers truncate/regrow churn; the
limit only fires after abnormal growth. No TRUNCATE checkpoint is introduced.

Scope: this is the *reclaim* half of DeusData#1083. It does not prevent the peak under
continuous reader contention (a checkpoint can't reset the log while a reader
holds a mark) — the write-amplification driver (concurrent full re-indexes of
one repo) is serialized by the cross-process pipeline lock in DeusData#892.

Test: a file-backed write store now reports journal_size_limit = 256 MiB
(RED on main: -1 / unlimited).

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>

Author: DeusData

src/store/store.c

@@ -432,6 +432,20 @@ static int configure_pragmas(cbm_store_t *s, bool in_memory, bool read_only) {
         if (rc != CBM_STORE_OK) {
             return rc;
         }
+        /* #1083: bound the WAL file so a checkpoint-starved log is physically
+         * reclaimed the next time a checkpoint can reset it. Our checkpoints are
+         * all PASSIVE (they never ftruncate — see cbm_store_checkpoint's SIGBUS
+         * note), so without a size limit the -wal file only ever grows; a
+         * journal_size_limit truncates it back to N bytes on the next successful
+         * reset. N is far above the healthy WAL (~4 MiB under the default
+         * 1000-page autocheckpoint), so normal indexing never triggers
+         * truncate/regrow churn — it only fires after abnormal growth. We do NOT
+         * use a TRUNCATE checkpoint: its ftruncate(fd,0) can raise SIGBUS in a
+         * sibling process that has the DB mmap'd on macOS. */
+        rc = exec_sql(s, "PRAGMA journal_size_limit = 268435456;"); /* 256 MiB */
+        if (rc != CBM_STORE_OK) {
+            return rc;
+        }
         char mmap_sql[ST_BUF_64];
         snprintf(mmap_sql, sizeof(mmap_sql), "PRAGMA mmap_size = %lld;",
                  (long long)cbm_store_resolve_mmap_size());
@@ -1081,6 +1095,25 @@ int cbm_store_checkpoint(cbm_store_t *s) {
     return exec_sql(s, "PRAGMA optimize;");
 }
 
+/* #1083: the WAL size limit configured on this (write) connection, in bytes.
+ * -1 means unlimited (SQLite's default — the pre-fix behavior). Per-connection
+ * and not persisted, so it can only be read on the connection that set it. */
+int64_t cbm_store_journal_size_limit(cbm_store_t *s) {
+    if (!s || !s->db) {
+        return -1;
+    }
+    sqlite3_stmt *stmt = NULL;
+    if (sqlite3_prepare_v2(s->db, "PRAGMA journal_size_limit;", -1, &stmt, NULL) != SQLITE_OK) {
+        return -1;
+    }
+    int64_t limit = -1;
+    if (sqlite3_step(stmt) == SQLITE_ROW) {
+        limit = sqlite3_column_int64(stmt, 0);
+    }
+    sqlite3_finalize(stmt);
+    return limit;
+}
+
 /* ── Dump ───────────────────────────────────────────────────────── */
 
 /* Dump entire in-memory database to a file via sqlite3_backup.

src/store/store.h

@@ -262,6 +262,10 @@ int cbm_store_create_indexes(cbm_store_t *s);
 /* Force WAL checkpoint + PRAGMA optimize. */
 int cbm_store_checkpoint(cbm_store_t *s);
 
+/* #1083: the WAL size limit (journal_size_limit) applied to this write
+ * connection, in bytes; -1 = unlimited (SQLite default / pre-fix). */
+int64_t cbm_store_journal_size_limit(cbm_store_t *s);
+
 /* Resolve the mmap_size pragma value applied to on-disk stores from the
  * CBM_SQLITE_MMAP_SIZE environment variable. Defaults to 67108864 (64 MB)
  * when the variable is unset, malformed, or partially numeric. Negative

tests/test_store_pragmas.c

@@ -86,6 +86,33 @@ TEST(store_open_with_mmap_disabled) {
     PASS();
 }
 
+/* #1083: on-disk write connections must bound the WAL via journal_size_limit
+ * so a checkpoint-starved log is physically reclaimed once a checkpoint can
+ * reset it. On main this is UNSET (-1 = unlimited), so the -wal file only ever
+ * grows (all our checkpoints are PASSIVE and never ftruncate). Read the pragma
+ * back on the SAME connection — it's per-connection and not persisted. */
+TEST(journal_size_limit_bounds_wal_issue1083) {
+    char tmp_path[256];
+    snprintf(tmp_path, sizeof(tmp_path), "%s/cbm_test_jsl_%d.db", cbm_tmpdir(), (int)getpid());
+    unlink(tmp_path);
+
+    cbm_store_t *s = cbm_store_open_path(tmp_path);
+    ASSERT(s != NULL);
+    /* 256 MiB — far above the healthy WAL (~4 MiB), so no truncate/regrow churn
+     * in normal operation; it only fires after abnormal (starved) growth. */
+    ASSERT(cbm_store_journal_size_limit(s) == (int64_t)268435456);
+    cbm_store_close(s);
+
+    unlink(tmp_path);
+    char tmp_wal[300];
+    char tmp_shm[300];
+    snprintf(tmp_wal, sizeof(tmp_wal), "%s-wal", tmp_path);
+    snprintf(tmp_shm, sizeof(tmp_shm), "%s-shm", tmp_path);
+    unlink(tmp_wal);
+    unlink(tmp_shm);
+    PASS();
+}
+
 
 /* #896: a row-scan that dies mid-stream (SQLITE_CORRUPT) must surface a
  * loud store error, not masquerade as a clean end of results. Counts are
@@ -179,6 +206,7 @@ TEST(corrupt_page_scan_returns_error_not_truncation) {
 }
 
 SUITE(store_pragmas) {
+    RUN_TEST(journal_size_limit_bounds_wal_issue1083);
     RUN_TEST(corrupt_page_scan_returns_error_not_truncation);
     RUN_TEST(mmap_size_default_when_unset);
     RUN_TEST(mmap_size_zero_disables_mmap);