Skip to content

Commit bad3e48

Browse files
authored
Merge pull request DeusData#2469 from DeusData/fix/issue-1419-sqlite-write-amplification
fix(store): cut WAL write amplification on 64 KiB-page indexes (DeusData#1419)
2 parents b5649e9 + eb99608 commit bad3e48

5 files changed

Lines changed: 229 additions & 13 deletions

File tree

‎src/pipeline/pipeline.c‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2267,10 +2267,15 @@ int cbm_pipeline_publish_staged(char *stage_path, const cbm_pipeline_generation_
22672267
free(stage_path);
22682268
return CBM_PIPELINE_PERSIST_FAILED;
22692269
}
2270-
bool ok = cbm_store_exec(store, "PRAGMA synchronous=FULL;") == CBM_STORE_OK;
2271-
ok = ok && cbm_store_delete_file_hashes(store, generation->project) == CBM_STORE_OK &&
2272-
cbm_store_upsert_file_hash_batch(store, generation->manifest,
2273-
generation->manifest_count) == CBM_STORE_OK;
2270+
/* No synchronous=FULL for these writes (#1419): the stage is private until
2271+
* the atomic rename and a crash discards it, so an fsync per WAL commit
2272+
* protects nothing here. The store's NORMAL level is SQLite's
2273+
* corruption-safe setting under WAL, and cbm_store_seal_for_atomic_publish()
2274+
* raises this connection to FULL for the checkpoint that makes the
2275+
* published file durable. */
2276+
bool ok = cbm_store_delete_file_hashes(store, generation->project) == CBM_STORE_OK &&
2277+
cbm_store_upsert_file_hash_batch(store, generation->manifest,
2278+
generation->manifest_count) == CBM_STORE_OK;
22742279
/* LSP surfaces belong to the generation: written inside the same staging
22752280
* store, before the atomic rename, so graph and surface data can never
22762281
* publish separately. The delete guards the incremental path, whose

‎src/store/store.c‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -541,6 +541,16 @@ int64_t cbm_store_resolve_mmap_size(void) {
541541
return (int64_t)parsed;
542542
}
543543

544+
/* #1419: page cache for read-write on-disk connections, in SQLite's negative
545+
* KiB form. sqlite_writer.c gives every index 64 KiB pages, so SQLite's
546+
* default 2000 KiB holds only ~31 of them, while a delta's purge and patch
547+
* dirty pages across the node and edge tables and all of their indexes. The
548+
* transaction then spills dirty pages into the WAL and rewrites them over and
549+
* over (5.2 GiB of WAL writes for a one-line change on a 294 MB django index).
550+
* 64 MiB = 1024 pages, the budget bulk mode already used; the cache only grows
551+
* as pages are touched. Query connections keep the default. */
552+
static const char ST_WRITE_CACHE_SQL[] = "PRAGMA cache_size = -65536;";
553+
544554
/* Configure connection pragmas.
545555
* in_memory — :memory: DB (synchronous OFF, no journal file).
546556
* read_only — query-only connection opened SQLITE_OPEN_READONLY. Runs
@@ -596,8 +606,9 @@ static int configure_pragmas(cbm_store_t *s, bool in_memory, bool read_only) {
596606
* cbm_store_checkpoint's SIGBUS note), so without a size limit the -wal
597607
* file only ever grows; journal_size_limit truncates it back to N bytes
598608
* on the next successful
599-
* reset. N is far above the healthy WAL (~4 MiB under the default
600-
* 1000-page autocheckpoint), so normal indexing never triggers
609+
* reset. N is far above the healthy WAL (~64 MiB: the default
610+
* 1000-page autocheckpoint at the 64 KiB page size sqlite_writer.c
611+
* gives every index), so normal indexing never triggers
601612
* truncate/regrow churn — it only fires after abnormal growth.
602613
* Shared/live paths do NOT use a TRUNCATE checkpoint: truncating the WAL
603614
* to zero can raise SIGBUS in a sibling process that has the DB mmap'd
@@ -606,6 +617,10 @@ static int configure_pragmas(cbm_store_t *s, bool in_memory, bool read_only) {
606617
if (rc != CBM_STORE_OK) {
607618
return rc;
608619
}
620+
rc = exec_sql(s, ST_WRITE_CACHE_SQL);
621+
if (rc != CBM_STORE_OK) {
622+
return rc;
623+
}
609624
char mmap_sql[ST_BUF_64];
610625
snprintf(mmap_sql, sizeof(mmap_sql), "PRAGMA mmap_size = %lld;",
611626
(long long)cbm_store_resolve_mmap_size());
@@ -1813,20 +1828,21 @@ int cbm_store_begin_bulk(cbm_store_t *s) {
18131828
* because the in-memory rollback journal is lost on crash.
18141829
* WAL mode is crash-safe: uncommitted WAL entries are simply discarded
18151830
* on the next open. Performance is preserved via synchronous=OFF and a
1816-
* larger cache, which are safe with WAL. */
1831+
* 64 MiB cache, which are safe with WAL. */
18171832
int rc = exec_sql(s, "PRAGMA synchronous = OFF;");
18181833
if (rc != CBM_STORE_OK) {
18191834
return rc;
18201835
}
1821-
return exec_sql(s, "PRAGMA cache_size = -65536;"); /* CBM_SZ_64 MB */
1836+
return exec_sql(s, ST_WRITE_CACHE_SQL);
18221837
}
18231838

18241839
int cbm_store_end_bulk(cbm_store_t *s) {
18251840
int rc = exec_sql(s, "PRAGMA synchronous = NORMAL;");
18261841
if (rc != CBM_STORE_OK) {
18271842
return rc;
18281843
}
1829-
return exec_sql(s, "PRAGMA cache_size = -2000;"); /* default ~2 MB */
1844+
/* Back to the read-write cache, not SQLite's 2000 KiB default (#1419). */
1845+
return exec_sql(s, ST_WRITE_CACHE_SQL);
18301846
}
18311847

18321848
int cbm_store_drop_indexes(cbm_store_t *s) {

‎src/store/store.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -434,11 +434,12 @@ int cbm_store_rollback(cbm_store_t *s);
434434

435435
/* ── Bulk write optimization ────────────────────────────────────── */
436436

437-
/* Tune pragmas for bulk write throughput (synchronous=OFF, large cache).
437+
/* Tune pragmas for bulk write throughput (synchronous=OFF, 64 MiB cache).
438438
* WAL journal mode is preserved throughout for crash safety. */
439439
int cbm_store_begin_bulk(cbm_store_t *s);
440440

441-
/* Restore normal pragmas (synchronous=NORMAL, default cache) after bulk writes. */
441+
/* Restore normal pragmas (synchronous=NORMAL, the 64 MiB read-write cache)
442+
* after bulk writes. */
442443
int cbm_store_end_bulk(cbm_store_t *s);
443444

444445
/* Drop user indexes for faster bulk inserts. */

‎tests/test_pipeline.c‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2945,6 +2945,115 @@ TEST(pipeline_incremental_repoints_call_reference_without_stale_edge) {
29452945
PASS();
29462946
}
29472947

2948+
/* #1419: every index publishes through a private staging DB in WAL mode that
2949+
* cbm_store_seal_for_atomic_publish() checkpoints at synchronous=FULL and
2950+
* switches to DELETE mode before the atomic rename. A crash before that rename
2951+
* discards the stage, so a WAL commit at FULL only adds an fsync per
2952+
* transaction without protecting anything; SQLite documents NORMAL as
2953+
* corruption-safe under WAL. The probe reads the EFFECTIVE synchronous level
2954+
* of the committing handle from a WAL hook on every connection the run opens.
2955+
*
2956+
* SQLite installs its default autocheckpoint WAL hook after auto-extensions
2957+
* run, so the auto-extension installs a statement trace and the trace installs
2958+
* the probe once the connection is fully open. The probe keeps SQLite's
2959+
* default 1000-frame autocheckpoint, so it does not change what the run
2960+
* writes. */
2961+
enum { I1419_SYNC_FULL = 2, I1419_DEFAULT_AUTOCHECKPOINT = 1000 };
2962+
static atomic_int g_i1419_wal_commits;
2963+
static atomic_int g_i1419_full_sync_commits;
2964+
2965+
static int i1419_wal_probe(void *arg, sqlite3 *db, const char *db_name, int wal_frames) {
2966+
(void)arg;
2967+
atomic_fetch_add(&g_i1419_wal_commits, 1);
2968+
sqlite3_stmt *stmt = NULL;
2969+
if (sqlite3_prepare_v2(db, "PRAGMA synchronous;", -1, &stmt, NULL) == SQLITE_OK &&
2970+
sqlite3_step(stmt) == SQLITE_ROW && sqlite3_column_int(stmt, 0) >= I1419_SYNC_FULL) {
2971+
atomic_fetch_add(&g_i1419_full_sync_commits, 1);
2972+
}
2973+
sqlite3_finalize(stmt);
2974+
if (wal_frames >= I1419_DEFAULT_AUTOCHECKPOINT) {
2975+
(void)sqlite3_wal_checkpoint(db, db_name);
2976+
}
2977+
return SQLITE_OK;
2978+
}
2979+
2980+
static int i1419_trace(unsigned type, void *ctx, void *stmt, void *sql) {
2981+
(void)type;
2982+
(void)ctx;
2983+
(void)sql;
2984+
(void)sqlite3_wal_hook(sqlite3_db_handle((sqlite3_stmt *)stmt), i1419_wal_probe, NULL);
2985+
return 0;
2986+
}
2987+
2988+
static int i1419_install_probe(sqlite3 *db, char **errmsg, const sqlite3_api_routines *api) {
2989+
(void)errmsg;
2990+
(void)api;
2991+
return sqlite3_trace_v2(db, SQLITE_TRACE_STMT, i1419_trace, NULL);
2992+
}
2993+
2994+
static int i1419_index(const char *repo, const char *db_path) {
2995+
cbm_pipeline_t *p = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL);
2996+
if (!p) {
2997+
return -1;
2998+
}
2999+
int rc = cbm_pipeline_run(p);
3000+
cbm_pipeline_free(p);
3001+
return rc;
3002+
}
3003+
3004+
TEST(pipeline_staged_publish_commits_below_full_sync_issue1419) {
3005+
char tmp[256];
3006+
snprintf(tmp, sizeof(tmp), "/tmp/cbm_i1419_XXXXXX");
3007+
if (!cbm_mkdtemp(tmp)) {
3008+
FAIL("tmpdir");
3009+
}
3010+
/* Equal lengths + equal mtimes: the second run must route on content
3011+
* bytes to the closure-repair (delta) publish, not a full rebuild. */
3012+
static const char initial_source[] =
3013+
"package syncprobe\n"
3014+
"func alphaSyncTarget() {}\n"
3015+
"func bravoSyncTarget() {}\n"
3016+
"func syncProbeAccept(callback func()) {}\n"
3017+
"func syncProbeSite() { syncProbeAccept(alphaSyncTarget) }\n";
3018+
static const char replacement_source[] =
3019+
"package syncprobe\n"
3020+
"func alphaSyncTarget() {}\n"
3021+
"func bravoSyncTarget() {}\n"
3022+
"func syncProbeAccept(callback func()) {}\n"
3023+
"func syncProbeSite() { syncProbeAccept(bravoSyncTarget) }\n";
3024+
ASSERT_EQ(sizeof(initial_source), sizeof(replacement_source));
3025+
write_temp_file(tmp, "probe.go", initial_source);
3026+
char source_path[512];
3027+
snprintf(source_path, sizeof(source_path), "%s/probe.go", tmp);
3028+
const time_t fixed_mtime = 1700000000;
3029+
ASSERT_EQ(pipeline_test_set_mtime(source_path, fixed_mtime, 123456789L), 0);
3030+
char db_path[512];
3031+
snprintf(db_path, sizeof(db_path), "%s/i1419.db", tmp);
3032+
3033+
atomic_store(&g_i1419_wal_commits, 0);
3034+
atomic_store(&g_i1419_full_sync_commits, 0);
3035+
ASSERT_EQ(sqlite3_auto_extension((void (*)(void))i1419_install_probe), SQLITE_OK);
3036+
/* Both publish routes run before any assertion, so the probe is always
3037+
* unregistered again even when a run fails. */
3038+
int full_rc = i1419_index(tmp, db_path);
3039+
write_temp_file(tmp, "probe.go", replacement_source);
3040+
int mtime_rc = pipeline_test_set_mtime(source_path, fixed_mtime, 123456789L);
3041+
cbm_pipeline_incremental_test_reset_faults();
3042+
int delta_rc = i1419_index(tmp, db_path);
3043+
cbm_incremental_route_t delta_route = cbm_pipeline_incremental_test_last_route();
3044+
(void)sqlite3_cancel_auto_extension((void (*)(void))i1419_install_probe);
3045+
th_rmtree(tmp);
3046+
3047+
ASSERT_EQ(full_rc, 0);
3048+
ASSERT_EQ(mtime_rc, 0);
3049+
ASSERT_EQ(delta_rc, 0);
3050+
ASSERT_EQ(delta_route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);
3051+
/* Non-vacuous: both publishes commit through the WAL. */
3052+
ASSERT_GT(atomic_load(&g_i1419_wal_commits), 0);
3053+
ASSERT_EQ(atomic_load(&g_i1419_full_sync_commits), 0);
3054+
PASS();
3055+
}
3056+
29483057
/* SQL DDL becomes first-class Table/View nodes wired into FROM/JOIN lineage,
29493058
* while the shared name registry must NOT leak those relations into other
29503059
* languages' textual resolution: a Python call or identifier sharing the
@@ -17292,6 +17401,7 @@ SUITE(pipeline) {
1729217401
SUITE(pipeline_semantic_manifest_repro) {
1729317402
RUN_TEST(incremental_downgrade_preserves_scope_and_artifact_across_change_noop_delete);
1729417403
RUN_TEST(pipeline_incremental_repoints_call_reference_without_stale_edge);
17404+
RUN_TEST(pipeline_staged_publish_commits_below_full_sync_issue1419);
1729517405
RUN_TEST(pipeline_sql_lineage_and_relation_isolation);
1729617406
RUN_TEST(pipeline_incremental_sql_table_rename_drops_stale_lineage);
1729717407
RUN_TEST(pipeline_dbt_jinja_lineage);

‎tests/test_store_pragmas.c‎

Lines changed: 86 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
#include "test_framework.h"
1212
#include "test_helpers.h"
1313
#include <store/store.h>
14+
#include "sqlite3.h" /* vendored/sqlite3 — read pragmas back on the store's own handle */
15+
#include <stdbool.h>
1416
#include <stdio.h>
1517
#include <stdlib.h>
1618
#include <string.h>
@@ -98,8 +100,9 @@ TEST(journal_size_limit_bounds_wal_issue1083) {
98100

99101
cbm_store_t *s = cbm_store_open_path(tmp_path);
100102
ASSERT(s != NULL);
101-
/* 256 MiB — far above the healthy WAL (~4 MiB), so no truncate/regrow churn
102-
* in normal operation; it only fires after abnormal (starved) growth. */
103+
/* 256 MiB — far above the healthy WAL (~64 MiB: 1000 autocheckpoint pages
104+
* of 64 KiB), so no truncate/regrow churn in normal operation; it only
105+
* fires after abnormal (starved) growth. */
103106
ASSERT(cbm_store_journal_size_limit(s) == (int64_t)268435456);
104107
cbm_store_close(s);
105108

@@ -113,6 +116,86 @@ TEST(journal_size_limit_bounds_wal_issue1083) {
113116
PASS();
114117
}
115118

119+
/* #1419: effective pragmas per store role, read on the handle itself —
120+
* synchronous is per-connection and never persisted, so an external sqlite3
121+
* shell reports its own default (FULL), not what the indexer runs with. */
122+
static int role_pragma_int(cbm_store_t *s, const char *sql) {
123+
sqlite3_stmt *stmt = NULL;
124+
int value = -1;
125+
if (sqlite3_prepare_v2(cbm_store_get_db(s), sql, -1, &stmt, NULL) == SQLITE_OK &&
126+
sqlite3_step(stmt) == SQLITE_ROW) {
127+
value = sqlite3_column_int(stmt, 0);
128+
}
129+
sqlite3_finalize(stmt);
130+
return value;
131+
}
132+
133+
static bool role_journal_mode_is(cbm_store_t *s, const char *want) {
134+
sqlite3_stmt *stmt = NULL;
135+
bool match = false;
136+
if (sqlite3_prepare_v2(cbm_store_get_db(s), "PRAGMA journal_mode;", -1, &stmt, NULL) ==
137+
SQLITE_OK &&
138+
sqlite3_step(stmt) == SQLITE_ROW) {
139+
const char *mode = (const char *)sqlite3_column_text(stmt, 0);
140+
match = mode && strcmp(mode, want) == 0;
141+
}
142+
sqlite3_finalize(stmt);
143+
return match;
144+
}
145+
146+
TEST(store_role_pragmas_issue1419) {
147+
enum {
148+
SYNC_OFF = 0,
149+
SYNC_NORMAL = 1,
150+
SYNC_FULL = 2,
151+
/* 64 MiB = 1024 of the 64 KiB pages every index is written with. */
152+
WRITE_CACHE_KIB = -65536,
153+
SQLITE_DEFAULT_CACHE_KIB = -2000,
154+
};
155+
char tmp_path[256];
156+
snprintf(tmp_path, sizeof(tmp_path), "%s/cbm_test_roles_%d.db", cbm_tmpdir(), (int)getpid());
157+
unlink(tmp_path);
158+
159+
/* Read-write role (live ADR writes, staging generations): WAL at NORMAL,
160+
* with a page cache that holds more than a few 64 KiB pages. */
161+
cbm_store_t *s = cbm_store_open_path(tmp_path);
162+
ASSERT(s != NULL);
163+
ASSERT_TRUE(role_journal_mode_is(s, "wal"));
164+
ASSERT_EQ(role_pragma_int(s, "PRAGMA synchronous;"), SYNC_NORMAL);
165+
ASSERT_EQ(role_pragma_int(s, "PRAGMA cache_size;"), WRITE_CACHE_KIB);
166+
/* Bulk role: sync off for the write burst, then back to the read-write
167+
* settings rather than SQLite's defaults. */
168+
ASSERT_EQ(cbm_store_begin_bulk(s), CBM_STORE_OK);
169+
ASSERT_EQ(role_pragma_int(s, "PRAGMA synchronous;"), SYNC_OFF);
170+
ASSERT_EQ(cbm_store_end_bulk(s), CBM_STORE_OK);
171+
ASSERT_EQ(role_pragma_int(s, "PRAGMA synchronous;"), SYNC_NORMAL);
172+
ASSERT_EQ(role_pragma_int(s, "PRAGMA cache_size;"), WRITE_CACHE_KIB);
173+
/* Seal role: the durable checkpoint before the atomic rename runs at FULL
174+
* and leaves a self-contained DELETE-mode file. */
175+
ASSERT_EQ(cbm_store_seal_for_atomic_publish(s), CBM_STORE_OK);
176+
ASSERT_EQ(role_pragma_int(s, "PRAGMA synchronous;"), SYNC_FULL);
177+
ASSERT_TRUE(role_journal_mode_is(s, "delete"));
178+
cbm_store_close(s);
179+
180+
/* Query role: read-only, never switches a sealed file back to WAL, and
181+
* keeps SQLite's small default cache (one per request, so it stays cheap). */
182+
cbm_store_t *q = cbm_store_open_path_query(tmp_path);
183+
ASSERT(q != NULL);
184+
ASSERT_EQ(sqlite3_db_readonly(cbm_store_get_db(q), "main"), 1);
185+
ASSERT_TRUE(role_journal_mode_is(q, "delete"));
186+
ASSERT_EQ(role_pragma_int(q, "PRAGMA cache_size;"), SQLITE_DEFAULT_CACHE_KIB);
187+
cbm_store_close(q);
188+
189+
unlink(tmp_path);
190+
char tmp_wal[300];
191+
char tmp_shm[300];
192+
snprintf(tmp_wal, sizeof(tmp_wal), "%s-wal", tmp_path);
193+
snprintf(tmp_shm, sizeof(tmp_shm), "%s-shm", tmp_path);
194+
unlink(tmp_wal);
195+
unlink(tmp_shm);
196+
PASS();
197+
}
198+
116199
/* Pagination-cursor generation: minted per DB file, bumped per index run.
117200
* Same store + reads only -> stable; upsert_project (every index run's choke
118201
* point) -> changes; two distinct DB files can never share a generation even
@@ -264,6 +347,7 @@ TEST(corrupt_page_scan_returns_error_not_truncation) {
264347

265348
SUITE(store_pragmas) {
266349
RUN_TEST(journal_size_limit_bounds_wal_issue1083);
350+
RUN_TEST(store_role_pragmas_issue1419);
267351
RUN_TEST(store_generation_tracks_mutations);
268352
RUN_TEST(store_generation_rejects_malformed_metadata_atomically);
269353
RUN_TEST(corrupt_page_scan_returns_error_not_truncation);

0 commit comments

Comments
 (0)