6060#include "foundation/hash_table.h"
6161#include "foundation/log.h"
6262#include "foundation/compat.h"
63+ #include "store/store.h"
6364#include "cbm.h"
65+ #include "sqlite3.h"
6466
6567#include <ctype.h>
6668#include <math.h>
7577 * Never reset here: callers snapshot before/after so nested runs compose. */
7678_Atomic uint64_t g_importance_nodes = 0 ;
7779_Atomic uint64_t g_importance_name_visits = 0 ;
80+ _Atomic uint64_t g_importance_store_rows = 0 ;
7881
7982enum {
8083 CBM_IMPORTANCE_GENERIC_MIN_FILES = 5 , /* name defined in >= N files -> generic */
@@ -212,75 +215,117 @@ static const char *imp_find_key(const char *json, const char *key) {
212215 return NULL ;
213216}
214217
215- /* Write the numeric "importance" key into a node's properties JSON object.
218+ /* Produce a copy of `json` carrying the numeric "importance" key set to
219+ * `score`, or NULL if `json` is not a JSON object (caller leaves it alone) or
220+ * on allocation failure. Caller owns the result.
216221 *
217- * IDEMPOTENT BY CONTRACT. The incremental path rehydrates nodes straight out
218- * of the store (cbm_gbuf_load_from_db), and those nodes ALREADY carry an
219- * "importance" key from the previous run. A pure append would emit
220- * {"importance":1.0,...,"importance":2.0} — silent property corruption that
221- * no build or schema check would catch. So: overwrite an existing key in
222- * place, append only when the key is genuinely absent.
222+ * IDEMPOTENT BY CONTRACT. Nodes reach this function already carrying an
223+ * "importance" key on every re-scoring route: the legacy-partial incremental
224+ * path rehydrates them via cbm_gbuf_load_from_db, and the store-level
225+ * recompute reads them straight out of SQLite. A pure append would emit
226+ * {"importance":1.0,...,"importance":2.0} — silent property corruption that no
227+ * build or schema check would catch. So: overwrite an existing key in place,
228+ * append only when the key is genuinely absent.
223229 *
224- * A properties_json that is not a JSON object is left untouched. */
225- void cbm_pipeline_importance_append_prop (cbm_gbuf_node_t * node , double score ) {
226- if (!node ) {
227- return ;
228- }
229- const char * old = node -> properties_json ? node -> properties_json : "{}" ;
230+ * SINGLE DEFINITION. Both scoring routes (the in-memory gbuf pass and the
231+ * SQL-level store recompute) write through this one function, so the JSON
232+ * shape cannot drift between them. */
233+ char * cbm_pipeline_importance_set_prop (const char * json , double score ) {
234+ const char * old = json ? json : "{}" ;
230235 size_t olen = strlen (old );
231236 if (olen < 2 || old [olen - 1 ] != '}' ) {
232- return ; /* not a JSON object — leave untouched */
237+ return NULL ; /* not a JSON object — leave untouched */
233238 }
234239
235240 char val [CBM_SZ_32 ];
236241 int vn = snprintf (val , sizeof (val ), "%.6f" , score );
237242 if (vn < 0 || (size_t )vn >= sizeof (val )) {
238- return ;
243+ return NULL ;
239244 }
240245
241246 const char * k = imp_find_key (old , "importance" );
242247 if (k ) {
243248 const char * colon = strchr (k , ':' );
244249 if (!colon ) {
245- return ; /* imp_find_key guarantees one, but never trust a raw scan */
250+ return NULL ; /* imp_find_key guarantees one, but never trust a raw scan */
246251 }
247252 const char * v = colon + 1 ;
248253 while (isspace ((unsigned char )* v )) {
249254 v ++ ;
250255 }
251256 const char * vend = imp_value_end (v );
252257 if (!vend ) {
253- return ;
258+ return NULL ;
254259 }
255260 size_t head = (size_t )(v - old );
256261 size_t tail = strlen (vend );
257262 char * neu = malloc (head + (size_t )vn + tail + 1 );
258263 if (!neu ) {
259- return ;
264+ return NULL ;
260265 }
261266 memcpy (neu , old , head );
262267 memcpy (neu + head , val , (size_t )vn );
263268 memcpy (neu + head + (size_t )vn , vend , tail + 1 );
264- free (node -> properties_json );
265- node -> properties_json = neu ;
266- return ;
269+ return neu ;
267270 }
268271
269272 char frag [CBM_SZ_64 ];
270273 int fn = snprintf (frag , sizeof (frag ), "%s\"importance\":%s}" , (olen == 2 ) ? "" : "," , val );
271274 if (fn < 0 || (size_t )fn >= sizeof (frag )) {
272- return ;
275+ return NULL ;
273276 }
274277 char * neu = malloc (olen - 1 + (size_t )fn + 1 );
275278 if (!neu ) {
276- return ;
279+ return NULL ;
277280 }
278281 memcpy (neu , old , olen - 1 ); /* copy without the trailing '}' */
279282 memcpy (neu + olen - 1 , frag , (size_t )fn + 1 );
283+ return neu ;
284+ }
285+
286+ /* gbuf-node convenience wrapper around cbm_pipeline_importance_set_prop. */
287+ void cbm_pipeline_importance_append_prop (cbm_gbuf_node_t * node , double score ) {
288+ if (!node ) {
289+ return ;
290+ }
291+ char * neu = cbm_pipeline_importance_set_prop (node -> properties_json , score );
292+ if (!neu ) {
293+ return ;
294+ }
280295 free (node -> properties_json );
281296 node -> properties_json = neu ;
282297}
283298
299+ /* ── The scoring rule ─────────────────────────────────────────────────
300+ *
301+ * SINGLE DEFINITION, deliberately. There are two routes that score nodes —
302+ * the in-memory gbuf pass (full index, legacy-partial incremental) and the
303+ * SQL-level recompute over the staging store (closure-delta incremental) —
304+ * and they gather their inputs very differently. Only the GATHERING differs;
305+ * the rule itself lives here alone, so the two routes cannot drift apart in
306+ * what a score means. The equivalence test in tests/test_importance.c guards
307+ * the remaining freedom: that both gatherings produce the same inputs for the
308+ * same graph. */
309+ double cbm_pipeline_importance_score (const cbm_importance_inputs_t * in ) {
310+ if (!in ) {
311+ return 0.0 ;
312+ }
313+ double score = sqrt ((double )(in -> num_refs > 0 ? in -> num_refs : 0 ));
314+ if (name_is_private (in -> name )) {
315+ score *= CBM_IMPORTANCE_PRIV_MUL ;
316+ }
317+ if (in -> name_distinct_files >= CBM_IMPORTANCE_GENERIC_MIN_FILES ) {
318+ score *= CBM_IMPORTANCE_GENERIC_MUL ;
319+ }
320+ if (name_is_distinctive (in -> name )) {
321+ score *= CBM_IMPORTANCE_DISTINCT_MUL ;
322+ }
323+ if (cbm_is_test_path (in -> file_path ) || in -> tests_target ) {
324+ score *= CBM_IMPORTANCE_TEST_MUL ;
325+ }
326+ return score ;
327+ }
328+
284329/* ── Distinct-file counting (the linear core) ─────────────────────────── */
285330
286331typedef struct {
@@ -367,24 +412,16 @@ void cbm_pipeline_pass_importance(cbm_pipeline_ctx_t *ctx) {
367412 for (int i = 0 ; i < count ; i ++ ) {
368413 cbm_gbuf_node_t * n = (cbm_gbuf_node_t * )nodes [i ];
369414
370- int num_refs =
371- incoming_edge_count (gb , n -> id , "CALLS" ) + incoming_edge_count (gb , n -> id , "USAGE" );
372- double score = sqrt ((double )num_refs );
373-
374- if (name_is_private (n -> name )) {
375- score *= CBM_IMPORTANCE_PRIV_MUL ;
376- }
377- if (imp_distinct_file_count (gb , n -> name , & ix ) >= CBM_IMPORTANCE_GENERIC_MIN_FILES ) {
378- score *= CBM_IMPORTANCE_GENERIC_MUL ;
379- }
380- if (name_is_distinctive (n -> name )) {
381- score *= CBM_IMPORTANCE_DISTINCT_MUL ;
382- }
383- if (cbm_is_test_path (n -> file_path ) || incoming_edge_count (gb , n -> id , "TESTS" ) > 0 ) {
384- score *= CBM_IMPORTANCE_TEST_MUL ;
385- }
386-
387- cbm_pipeline_importance_append_prop (n , score );
415+ /* Gather only — the rule itself lives in one place. */
416+ cbm_importance_inputs_t in = {
417+ .name = n -> name ,
418+ .file_path = n -> file_path ,
419+ .num_refs = incoming_edge_count (gb , n -> id , "CALLS" ) +
420+ incoming_edge_count (gb , n -> id , "USAGE" ),
421+ .name_distinct_files = imp_distinct_file_count (gb , n -> name , & ix ),
422+ .tests_target = incoming_edge_count (gb , n -> id , "TESTS" ) > 0 ,
423+ };
424+ cbm_pipeline_importance_append_prop (n , cbm_pipeline_importance_score (& in ));
388425 updated ++ ;
389426 }
390427 }
@@ -394,3 +431,126 @@ void cbm_pipeline_pass_importance(cbm_pipeline_ctx_t *ctx) {
394431 atomic_fetch_add_explicit (& g_importance_nodes , updated , memory_order_relaxed );
395432 cbm_log_info ("pass.importance" , "symbols" , itoa_imp (updated ));
396433}
434+
435+ /* ── Store-level recompute (closure-delta incremental) ────────────────
436+ *
437+ * WHY THIS EXISTS. The closure-delta route never materialises the project
438+ * graph in RAM: cbm_delta_preseed fills the delta gbuf with *proxy* nodes
439+ * (id, label, name, qn, file_path — no edges, empty properties), and only the
440+ * re-extracted files' symbols carry real edges. Scoring there reads a
441+ * near-zero in-degree for a symbol referenced hundreds of times project-wide,
442+ * and cbm_delta_patch then persists exactly those fresh rows. Loading the full
443+ * graph to fix that would give back the point of the delta route, so the
444+ * recompute runs where the complete graph already exists: in SQL, over the
445+ * staging store, AFTER cbm_delta_patch has inserted the new nodes and
446+ * re-linked the snapshotted inbound edges.
447+ *
448+ * WHY IT CANNOT DRIFT FROM THE IN-MEMORY PASS. The scoring rule and the JSON
449+ * write-back are not reimplemented here. They are registered as SQLite
450+ * user-defined functions that call cbm_pipeline_importance_score() and
451+ * cbm_pipeline_importance_set_prop() — the same C code the gbuf pass runs.
452+ * SQL supplies only the three aggregates (in-degree, TESTS presence, distinct
453+ * files per name), each one indexed GROUP BY. What remains free is the
454+ * GATHERING, and the full-vs-incremental equivalence test guards that.
455+ *
456+ * The whole project is rescored, not just the changed files: distinct-files-
457+ * per-name is a global input, so editing one file can legitimately move a name
458+ * across the generic-name threshold for every other definition of that name. A
459+ * scoped recompute would leave those stale — and would fail the equivalence
460+ * test, correctly. */
461+
462+ /* UDF: cbm_imp_score(name, file_path, num_refs, name_files, tests_target). */
463+ static void sql_importance_score (sqlite3_context * c , int argc , sqlite3_value * * argv ) {
464+ if (argc != 5 ) {
465+ sqlite3_result_error (c , "cbm_imp_score arity" , -1 );
466+ return ;
467+ }
468+ cbm_importance_inputs_t in = {
469+ .name = (const char * )sqlite3_value_text (argv [0 ]),
470+ .file_path = (const char * )sqlite3_value_text (argv [1 ]),
471+ .num_refs = sqlite3_value_int (argv [2 ]),
472+ .name_distinct_files = sqlite3_value_int (argv [3 ]),
473+ .tests_target = sqlite3_value_int (argv [4 ]) != 0 ,
474+ };
475+ sqlite3_result_double (c , cbm_pipeline_importance_score (& in ));
476+ }
477+
478+ /* UDF: cbm_imp_set(properties, score) → properties with "importance" set.
479+ * A non-object blob yields NULL; the UPDATE's COALESCE then leaves that row
480+ * untouched — the same tolerance the in-memory writer has. */
481+ static void sql_importance_set (sqlite3_context * c , int argc , sqlite3_value * * argv ) {
482+ if (argc != 2 ) {
483+ sqlite3_result_error (c , "cbm_imp_set arity" , -1 );
484+ return ;
485+ }
486+ const char * props = (const char * )sqlite3_value_text (argv [0 ]);
487+ char * neu = cbm_pipeline_importance_set_prop (props , sqlite3_value_double (argv [1 ]));
488+ if (!neu ) {
489+ sqlite3_result_null (c );
490+ return ;
491+ }
492+ sqlite3_result_text (c , neu , -1 , SQLITE_TRANSIENT );
493+ free (neu );
494+ }
495+
496+ /* The aggregate CTE is MATERIALIZED on purpose: it reads `nodes` while the
497+ * UPDATE writes `nodes`, and materialising settles that ordering explicitly
498+ * rather than relying on the planner to choose it. */
499+ static const char * const kImportanceRecomputeSQL =
500+ "WITH agg AS MATERIALIZED ("
501+ " SELECT n.id AS id,"
502+ " cbm_imp_score(n.name, n.file_path, COALESCE(d.refs, 0),"
503+ " COALESCE(f.files, 0), COALESCE(d.tests, 0)) AS score"
504+ " FROM nodes n"
505+ " LEFT JOIN (SELECT target_id AS id,"
506+ " SUM(CASE WHEN type IN ('CALLS','USAGE') THEN 1 ELSE 0 END) AS refs,"
507+ " SUM(CASE WHEN type = 'TESTS' THEN 1 ELSE 0 END) AS tests"
508+ " FROM edges WHERE project = ?1 GROUP BY target_id) d ON d.id = n.id"
509+ " LEFT JOIN (SELECT name, COUNT(DISTINCT file_path) AS files"
510+ " FROM nodes WHERE project = ?1 GROUP BY name) f ON f.name = n.name"
511+ " WHERE n.project = ?1 AND n.label IN ('Function','Method','Class')"
512+ ")"
513+ "UPDATE nodes SET properties = COALESCE(cbm_imp_set(properties, agg.score), properties)"
514+ " FROM agg WHERE nodes.id = agg.id" ;
515+
516+ int cbm_pipeline_importance_recompute_store (cbm_store_t * store , const char * project ) {
517+ if (!store || !project ) {
518+ return -1 ;
519+ }
520+ sqlite3 * db = cbm_store_get_db (store );
521+ if (!db ) {
522+ return -1 ;
523+ }
524+ if (sqlite3_create_function (db , "cbm_imp_score" , 5 , SQLITE_UTF8 | SQLITE_DETERMINISTIC , NULL ,
525+ sql_importance_score , NULL , NULL ) != SQLITE_OK ||
526+ sqlite3_create_function (db , "cbm_imp_set" , 2 , SQLITE_UTF8 | SQLITE_DETERMINISTIC , NULL ,
527+ sql_importance_set , NULL , NULL ) != SQLITE_OK ) {
528+ cbm_log_error ("importance.store_udf_failed" , "err" , sqlite3_errmsg (db ));
529+ return -1 ;
530+ }
531+ if (cbm_store_begin (store ) != CBM_STORE_OK ) {
532+ return -1 ;
533+ }
534+ sqlite3_stmt * st = NULL ;
535+ if (sqlite3_prepare_v2 (db , kImportanceRecomputeSQL , -1 , & st , NULL ) != SQLITE_OK ) {
536+ cbm_log_error ("importance.store_prepare_failed" , "err" , sqlite3_errmsg (db ));
537+ cbm_store_rollback (store );
538+ return -1 ;
539+ }
540+ sqlite3_bind_text (st , 1 , project , -1 , SQLITE_TRANSIENT );
541+ int step_rc = sqlite3_step (st );
542+ sqlite3_finalize (st );
543+ if (step_rc != SQLITE_DONE ) {
544+ cbm_log_error ("importance.store_step_failed" , "err" , sqlite3_errmsg (db ));
545+ cbm_store_rollback (store );
546+ return -1 ;
547+ }
548+ int changed = sqlite3_changes (db );
549+ if (cbm_store_commit (store ) != CBM_STORE_OK ) {
550+ return -1 ;
551+ }
552+ uint64_t rows = (uint64_t )(changed > 0 ? changed : 0 );
553+ atomic_fetch_add_explicit (& g_importance_store_rows , rows , memory_order_relaxed );
554+ cbm_log_info ("pass.importance_store" , "symbols" , itoa_imp (rows ));
555+ return 0 ;
556+ }
0 commit comments