Skip to content

Commit abfac06

Browse files
committed
perf(pipeline): narrow the delta proxy set; map unproxied symbols at patch time
Two changes that move the delta path toward O(change) without touching the parallel-resolve contract. Proxy narrowing. The delta executor pre-loaded every project node so resolution could find cross-file targets by qualified name. Nodes that resolution can never look up are pure load cost, and on the kernel they dominate: six of its 8.5M nodes are Macro. The set is narrowed by EXCLUSION rather than an inclusion list, deliberately -- an earlier inclusion list was disproven by counterexamples it did not anticipate (synthetic Decorator nodes, then Macro), and excluding a short list of labels that are never lookup targets fails safe where guessing the full inclusion set did not. Patch-time identity mapping. A resolver that upserts a symbol the narrowed set did not pre-load now produces a stand-in node; the patch maps it back onto its existing row by qualified name instead of raising the UNIQUE violation the previous patch would have. Nodes from CHANGED files cannot collide here -- the purge removed them -- so repaired files still receive fresh rows. The map is a sorted array searched by bisection, not a CBMHashTable: that table stores key POINTERS without copying them, which a stack-formatted integer key cannot satisfy. NOT attempted here, and recorded instead: making the proxy load itself lazy. cbm_parallel_resolve documents main_gbuf as READ-ONLY during its worker phase and its workers do call cbm_gbuf_find_by_qn on it, so a find-time materializer would mutate a buffer under concurrent readers. A safe version needs the materialization hoisted ahead of the worker phase; that is a separate change with its own verification. Kernel one-file warm: 50.8s -> 42.7s wall, peak RSS 13.2 -> 10.8GB, proxies 8.5M -> 2.5M (preseed 18.3s -> 9.7s), and the run reports remapped=0 -- no resolver needed a symbol the narrowing dropped. 535 pipeline/incremental/store/cross-repo/integration tests green, including the full convergence matrix. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 708603a commit abfac06

1 file changed

Lines changed: 103 additions & 6 deletions

File tree

‎src/pipeline/pipeline_delta.c‎

Lines changed: 103 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -250,10 +250,25 @@ int64_t cbm_delta_preseed(cbm_store_t *store, const char *project, cbm_gbuf_t *g
250250
sqlite3_finalize(stmt);
251251
}
252252

253+
/* Proxy only what resolution can reach by NAME or QN: the registry
254+
* symbols (cbm_label_is_type_like + Function/Method/Variable/Field, the
255+
* exact set incr_label_is_registry_symbol admits) plus the structural
256+
* nodes edges attach to. Everything else -- overwhelmingly Macro, six of
257+
* the kernel's 8.5M nodes -- is never a lookup target, and a proxy for it
258+
* would be pure load cost.
259+
*
260+
* Safety net for the labels this list does not anticipate: a resolver
261+
* that upserts an unproxied QN now produces a fresh node that the patch
262+
* maps back onto its existing row by qualified name, rather than the
263+
* UNIQUE violation the pre-remap patch would have raised. A resolver that
264+
* only LOOKS UP still needs its target resident, which is why the list
265+
* mirrors the registry's own membership rule instead of guessing. */
253266
sqlite3_stmt *stmt = NULL;
254267
if (sqlite3_prepare_v2(db,
255268
"SELECT id, label, name, qualified_name, file_path FROM nodes"
256-
" WHERE project = ?1 ORDER BY id",
269+
" WHERE project = ?1 AND label NOT IN"
270+
" ('Macro','Comment','Section','Branch','Commit','Tag')"
271+
" ORDER BY id",
257272
CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
258273
return -1;
259274
}
@@ -296,16 +311,81 @@ typedef struct {
296311
int64_t max_db_id;
297312
sqlite3_stmt *node_stmt;
298313
sqlite3_stmt *edge_stmt;
314+
sqlite3_stmt *qn_lookup;
315+
/* gbuf id -> real database id for any fresh node whose qualified
316+
* name turns out to already exist on disk: a resolver referenced
317+
* a symbol the narrowed proxy set did not pre-load, so it upserted
318+
* a stand-in. Mapping it back keeps the existing row authoritative
319+
* and every edge endpoint database-valid. Appended in ascending
320+
* gbuf-id order (nodes are visited that way), so lookups binary
321+
* search -- CBMHashTable stores key POINTERS without copying,
322+
* which a stack-formatted integer key cannot satisfy. */
323+
struct {
324+
int64_t from;
325+
int64_t to;
326+
} *remap;
327+
int remap_count;
328+
int remap_cap;
299329
bool failed;
300330
int64_t nodes;
301331
int64_t edges;
332+
int64_t remapped;
302333
} delta_patch_ctx_t;
303334

335+
static void delta_remap_put(delta_patch_ctx_t *ctx, int64_t from, int64_t to) {
336+
if (ctx->remap_count >= ctx->remap_cap) {
337+
int ncap = ctx->remap_cap ? ctx->remap_cap * 2 : 256;
338+
void *grown = realloc(ctx->remap, (size_t)ncap * sizeof(*ctx->remap));
339+
if (!grown) {
340+
ctx->failed = true;
341+
return;
342+
}
343+
ctx->remap = grown;
344+
ctx->remap_cap = ncap;
345+
}
346+
ctx->remap[ctx->remap_count].from = from;
347+
ctx->remap[ctx->remap_count].to = to;
348+
ctx->remap_count++;
349+
}
350+
351+
static int64_t delta_remap_get(const delta_patch_ctx_t *ctx, int64_t id) {
352+
if (id <= ctx->max_db_id || ctx->remap_count == 0) {
353+
return id;
354+
}
355+
int lo = 0;
356+
int hi = ctx->remap_count - 1;
357+
while (lo <= hi) {
358+
int mid = lo + (hi - lo) / 2;
359+
if (ctx->remap[mid].from == id) {
360+
return ctx->remap[mid].to;
361+
}
362+
if (ctx->remap[mid].from < id) {
363+
lo = mid + 1;
364+
} else {
365+
hi = mid - 1;
366+
}
367+
}
368+
return id;
369+
}
370+
304371
static void delta_patch_node(const cbm_gbuf_node_t *node, void *userdata) {
305372
delta_patch_ctx_t *ctx = (delta_patch_ctx_t *)userdata;
306373
if (ctx->failed || node->id <= ctx->max_db_id) {
307374
return;
308375
}
376+
if (node->qualified_name) {
377+
sqlite3_reset(ctx->qn_lookup);
378+
sqlite3_bind_text(ctx->qn_lookup, 1, ctx->project, CBM_NOT_FOUND, SQLITE_TRANSIENT);
379+
sqlite3_bind_text(ctx->qn_lookup, 2, node->qualified_name, CBM_NOT_FOUND, SQLITE_TRANSIENT);
380+
if (sqlite3_step(ctx->qn_lookup) == SQLITE_ROW) {
381+
/* Already on disk: an unchanged file's symbol. A node from a
382+
* CHANGED file cannot appear here -- the purge removed it --
383+
* so repaired files still receive fresh rows. */
384+
delta_remap_put(ctx, node->id, sqlite3_column_int64(ctx->qn_lookup, 0));
385+
ctx->remapped++;
386+
return;
387+
}
388+
}
309389
sqlite3_reset(ctx->node_stmt);
310390
sqlite3_bind_int64(ctx->node_stmt, 1, node->id);
311391
sqlite3_bind_text(ctx->node_stmt, 2, ctx->project, CBM_NOT_FOUND, SQLITE_TRANSIENT);
@@ -347,13 +427,16 @@ static void delta_patch_edge(const cbm_gbuf_edge_t *edge, void *userdata) {
347427
if (ctx->failed) {
348428
return;
349429
}
350-
if (edge->source_id <= ctx->max_db_id && edge->target_id <= ctx->max_db_id) {
351-
return; /* both endpoints pre-existed: nothing new to record */
430+
int64_t src = delta_remap_get(ctx, edge->source_id);
431+
int64_t tgt = delta_remap_get(ctx, edge->target_id);
432+
if (src <= ctx->max_db_id && tgt <= ctx->max_db_id && src == edge->source_id &&
433+
tgt == edge->target_id) {
434+
return; /* both endpoints pre-existed unremapped: already recorded */
352435
}
353436
sqlite3_reset(ctx->edge_stmt);
354437
sqlite3_bind_text(ctx->edge_stmt, 1, ctx->project, CBM_NOT_FOUND, SQLITE_TRANSIENT);
355-
sqlite3_bind_int64(ctx->edge_stmt, 2, edge->source_id);
356-
sqlite3_bind_int64(ctx->edge_stmt, 3, edge->target_id);
438+
sqlite3_bind_int64(ctx->edge_stmt, 2, src);
439+
sqlite3_bind_int64(ctx->edge_stmt, 3, tgt);
357440
sqlite3_bind_text(ctx->edge_stmt, 4, edge->type, CBM_NOT_FOUND, SQLITE_TRANSIENT);
358441
sqlite3_bind_text(ctx->edge_stmt, 5, edge->properties_json ? edge->properties_json : "{}",
359442
CBM_NOT_FOUND, SQLITE_TRANSIENT);
@@ -375,6 +458,13 @@ int cbm_delta_patch(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf, i
375458
return CBM_NOT_FOUND;
376459
}
377460
delta_patch_ctx_t ctx = {.db = db, .project = project, .max_db_id = max_db_id};
461+
if (sqlite3_prepare_v2(db,
462+
"SELECT id FROM nodes WHERE project = ?1"
463+
" AND qualified_name = ?2",
464+
CBM_NOT_FOUND, &ctx.qn_lookup, NULL) != SQLITE_OK) {
465+
cbm_store_rollback(store);
466+
return CBM_NOT_FOUND;
467+
}
378468
if (sqlite3_prepare_v2(db,
379469
"INSERT INTO nodes (id, project, label, name, qualified_name,"
380470
" file_path, start_line, end_line, properties)"
@@ -386,6 +476,8 @@ int cbm_delta_patch(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf, i
386476
CBM_NOT_FOUND, &ctx.edge_stmt, NULL) != SQLITE_OK) {
387477
sqlite3_finalize(ctx.node_stmt);
388478
sqlite3_finalize(ctx.edge_stmt);
479+
sqlite3_finalize(ctx.qn_lookup);
480+
free(ctx.remap);
389481
cbm_store_rollback(store);
390482
return CBM_NOT_FOUND;
391483
}
@@ -395,6 +487,7 @@ int cbm_delta_patch(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf, i
395487
}
396488
sqlite3_finalize(ctx.node_stmt);
397489
sqlite3_finalize(ctx.edge_stmt);
490+
sqlite3_finalize(ctx.qn_lookup);
398491

399492
/* Re-link the snapshotted inbound edges by qualified name. A target
400493
* whose QN no longer exists simply matches no row — full-reindex
@@ -451,16 +544,20 @@ int cbm_delta_patch(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf, i
451544
}
452545

453546
if (ctx.failed) {
547+
free(ctx.remap);
454548
cbm_store_rollback(store);
455549
return CBM_NOT_FOUND;
456550
}
551+
free(ctx.remap);
457552
if (cbm_store_commit(store) != CBM_STORE_OK) {
458553
return CBM_NOT_FOUND;
459554
}
460555
char nodes_buf[32];
461556
char edges_buf[32];
462557
snprintf(nodes_buf, sizeof(nodes_buf), "%lld", (long long)ctx.nodes);
463558
snprintf(edges_buf, sizeof(edges_buf), "%lld", (long long)ctx.edges);
464-
cbm_log_info("delta.patch", "nodes", nodes_buf, "edges", edges_buf);
559+
char remap_buf[32];
560+
snprintf(remap_buf, sizeof(remap_buf), "%lld", (long long)ctx.remapped);
561+
cbm_log_info("delta.patch", "nodes", nodes_buf, "edges", edges_buf, "remapped", remap_buf);
465562
return 0;
466563
}

0 commit comments

Comments
 (0)