2222#include "foundation/hash_table.h"
2323#include "foundation/log.h"
2424#include "foundation/compat.h"
25+ #include "foundation/compat_fs.h" /* cbm_fopen for the pair dump */
2526#define XXH_INLINE_ALL
2627#include "xxhash/xxhash.h"
2728
@@ -194,23 +195,33 @@ static int inject_callee_tokens(const char *name, char **tokens, int count, int
194195}
195196
196197/* Walk the outbound CALLS edges of n and inject tokens for each target. */
198+ static const char * * collect_sorted_call_neighbors (const cbm_gbuf_t * gbuf , int64_t node_id ,
199+ bool outbound , int * out_n );
200+
197201static int inject_calls_pattern_tokens (const cbm_gbuf_node_t * n , const cbm_gbuf_t * gbuf ,
198202 char * * tokens , int count , int max_tokens ) {
199203 if (!gbuf ) {
200204 return count ;
201205 }
202- const cbm_gbuf_edge_t * * edges = NULL ;
203- int ec = 0 ;
204- if (cbm_gbuf_find_edges_by_source_type (gbuf , n -> id , "CALLS" , & edges , & ec ) != 0 ) {
206+ /* SORTED, like every other consumer of CALLS neighbours in this file. The
207+ * raw edge array is in graph-buffer insertion order — parallel worker merge
208+ * order — and this loop both ORDERS the injected tokens and truncates at
209+ * max_tokens. Co-occurrence enrichment reads a window around each token
210+ * POSITION, so a different order yields different token vectors while
211+ * leaving document frequencies untouched: measured on the go corpus
212+ * 2026-09-19, two runs of one binary with identical idf but different token
213+ * sequences and different enriched vectors, which flickered near-threshold
214+ * SEMANTICALLY_RELATED edges. collect_sorted_call_neighbors() was written
215+ * for exactly this class and this call site was missed. */
216+ int nn = 0 ;
217+ const char * * names = collect_sorted_call_neighbors (gbuf , n -> id , /*outbound=*/ true, & nn );
218+ if (!names ) {
205219 return count ;
206220 }
207- for (int e = 0 ; e < ec && count < max_tokens ; e ++ ) {
208- const cbm_gbuf_node_t * t = cbm_gbuf_find_by_id (gbuf , edges [e ]-> target_id );
209- if (!t || !t -> name ) {
210- continue ;
211- }
212- count = inject_callee_tokens (t -> name , tokens , count , max_tokens );
221+ for (int e = 0 ; e < nn && count < max_tokens ; e ++ ) {
222+ count = inject_callee_tokens (names [e ], tokens , count , max_tokens );
213223 }
224+ cbm_free (CBM_MEM_CLASS_SEMANTIC , (void * )names );
214225 return count ;
215226}
216227
@@ -1152,9 +1163,27 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf
11521163 }
11531164 qsort (pairs , (size_t )n , sizeof (deferred_edge_t ), cmp_deferred_edge_canonical );
11541165
1166+ /* CBM_SEM_PAIR_DUMP=<path>: every candidate that passed the threshold, with
1167+ * its score, in canonical order. Written HERE because this loop is
1168+ * single-threaded and already sorted, so the dump needs no lock and two
1169+ * runs compare line by line. The (i, j) indices are stable across runs —
1170+ * funcs[] is sorted by qualified name — where node ids are not. Exists to
1171+ * chase the near-threshold SEMANTICALLY_RELATED flicker: find the pair that
1172+ * flips, then compare its score between runs. */
1173+ FILE * pair_dump = NULL ;
1174+ {
1175+ char dump_path [CBM_SZ_1K ];
1176+ if (cbm_safe_getenv ("CBM_SEM_PAIR_DUMP" , dump_path , sizeof (dump_path ), NULL )) {
1177+ pair_dump = cbm_fopen (dump_path , "w" );
1178+ }
1179+ }
1180+
11551181 int total_edges = 0 ;
11561182 for (int e = 0 ; e < n ; e ++ ) {
11571183 deferred_edge_t * de = & pairs [e ];
1184+ if (pair_dump ) {
1185+ fprintf (pair_dump , "%d %d %.9g\n" , de -> i , de -> j , (double )de -> score );
1186+ }
11581187 if (edge_counts [de -> i ] >= max_edges || edge_counts [de -> j ] >= max_edges ) {
11591188 continue ;
11601189 }
@@ -1166,6 +1195,9 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf
11661195 edge_counts [de -> j ]++ ;
11671196 total_edges ++ ;
11681197 }
1198+ if (pair_dump ) {
1199+ (void )fclose (pair_dump );
1200+ }
11691201 cbm_free (CBM_MEM_CLASS_SEMANTIC , pairs );
11701202 return total_edges ;
11711203}
@@ -1370,6 +1402,139 @@ static void phase5_lsh_build(cbm_sem_func_t *funcs, int func_count, int worker_c
13701402 * out_buckets = band_buckets ;
13711403}
13721404
1405+ /* Checksum every input the scorer reads, so two runs can be compared at the
1406+ * seam right before scoring. cbm_sem_combined_score() reads exactly these
1407+ * fields, so identical checksums with differing scores would be impossible.
1408+ * Placed AFTER tf-idf is filled (phase 4), not after phase 1b where the
1409+ * sparse vectors are still empty. Diagnostic, off unless the env var is set. */
1410+ static void sem_state_dump (const cbm_sem_func_t * funcs , int func_count ) {
1411+ /* CBM_SEM_STATE_DUMP=1: one checksum over every per-function vector, so two
1412+ * runs can be compared at THIS seam. Scores are pure math over these
1413+ * vectors, so if the checksums match and the scores do not, the difference
1414+ * is downstream; if they differ, it is here or above. Diagnostic only. */
1415+ char state_path [CBM_SZ_1K ];
1416+ if (cbm_safe_getenv ("CBM_SEM_STATE_DUMP" , state_path , sizeof (state_path ), NULL )) {
1417+ uint64_t h = 1469598103934665603ULL ;
1418+ uint64_t h_codes = 1469598103934665603ULL ;
1419+ uint64_t h_tfidf = 1469598103934665603ULL ;
1420+ for (int i = 0 ; i < func_count ; i ++ ) {
1421+ const unsigned char * bytes [] = {
1422+ (const unsigned char * )& funcs [i ].ri_code ,
1423+ (const unsigned char * )& funcs [i ].api_code ,
1424+ (const unsigned char * )& funcs [i ].type_code ,
1425+ (const unsigned char * )& funcs [i ].deco_code ,
1426+ };
1427+ for (size_t b = 0 ; b < sizeof (bytes ) / sizeof (bytes [0 ]); b ++ ) {
1428+ for (size_t k = 0 ; k < sizeof (cbm_rsq_code_t ); k ++ ) {
1429+ h ^= bytes [b ][k ];
1430+ h *= 1099511628211ULL ;
1431+ h_codes ^= bytes [b ][k ];
1432+ h_codes *= 1099511628211ULL ;
1433+ }
1434+ }
1435+ /* The other scoring inputs: sparse TF-IDF, the AST profile and the
1436+ * minhash fingerprint. Scoring reads exactly these plus the codes
1437+ * above, so a matching checksum means the scores must match too. */
1438+ const unsigned char * sp = (const unsigned char * )funcs [i ].struct_profile ;
1439+ for (size_t k = 0 ; k < sizeof (funcs [i ].struct_profile ); k ++ ) {
1440+ h ^= sp [k ];
1441+ h *= 1099511628211ULL ;
1442+ }
1443+ const unsigned char * mh = (const unsigned char * )funcs [i ].minhash ;
1444+ for (size_t k = 0 ; k < (funcs [i ].has_minhash ? sizeof (funcs [i ].minhash ) : 0 ); k ++ ) {
1445+ h ^= mh [k ];
1446+ h *= 1099511628211ULL ;
1447+ }
1448+ for (int w = 0 ; w < funcs [i ].tfidf_len ; w ++ ) {
1449+ h ^= (uint64_t )funcs [i ].tfidf_indices [w ];
1450+ h *= 1099511628211ULL ;
1451+ uint32_t bits ;
1452+ memcpy (& bits , & funcs [i ].tfidf_weights [w ], sizeof (bits ));
1453+ h ^= bits ;
1454+ h *= 1099511628211ULL ;
1455+ h_tfidf ^= (uint64_t )funcs [i ].tfidf_indices [w ];
1456+ h_tfidf *= 1099511628211ULL ;
1457+ h_tfidf ^= bits ;
1458+ h_tfidf *= 1099511628211ULL ;
1459+ }
1460+ }
1461+ /* To a FILE, not the log: the semantic pass runs in the index WORKER,
1462+ * whose log is removed when the run succeeds. */
1463+ FILE * sf = cbm_fopen (state_path , "w" );
1464+ if (sf ) {
1465+ fprintf (sf , "all=%016llx codes=%016llx tfidf=%016llx n=%d\n" , (unsigned long long )h ,
1466+ (unsigned long long )h_codes , (unsigned long long )h_tfidf , func_count );
1467+ (void )fclose (sf );
1468+ }
1469+ }
1470+ }
1471+
1472+ /* Running hash of the token SEQUENCE handed to the corpus: the set AND the
1473+ * order, because co-occurrence enrichment reads a window around each position,
1474+ * so the same tokens in a different order produce different vectors while
1475+ * leaving document frequencies (and therefore idf) untouched. Diagnostic for
1476+ * the SEMANTICALLY_RELATED flicker; off unless CBM_SEM_STATE_DUMP is set. */
1477+ static uint64_t g_sem_token_hash = 1469598103934665603ULL ;
1478+
1479+ static void sem_tokens_hash (char * * tokens , const size_t * offsets , const int * counts , int n ) {
1480+ char flag [CBM_SZ_1K ];
1481+ if (!cbm_safe_getenv ("CBM_SEM_STATE_DUMP" , flag , sizeof (flag ), NULL )) {
1482+ return ;
1483+ }
1484+ for (int d = 0 ; d < n ; d ++ ) {
1485+ for (int k = 0 ; k < counts [d ]; k ++ ) {
1486+ const char * tok = tokens [offsets [d ] + (size_t )k ];
1487+ for (const unsigned char * c = (const unsigned char * )tok ; c && * c ; c ++ ) {
1488+ g_sem_token_hash ^= * c ;
1489+ g_sem_token_hash *= 1099511628211ULL ;
1490+ }
1491+ g_sem_token_hash ^= 0xff ; /* token boundary */
1492+ g_sem_token_hash *= 1099511628211ULL ;
1493+ }
1494+ g_sem_token_hash ^= 0xfe ; /* document boundary */
1495+ g_sem_token_hash *= 1099511628211ULL ;
1496+ }
1497+ }
1498+
1499+ /* Checksum the corpus AFTER enrichment: the per-function vectors and tf-idf are
1500+ * both derived from it, so if this differs between two runs, everything
1501+ * downstream differs and the cause is in the corpus build rather than in
1502+ * scoring. Diagnostic, off unless CBM_SEM_STATE_DUMP is set. */
1503+ static void sem_corpus_dump (const cbm_sem_corpus_t * corpus , const char * suffix ) {
1504+ char state_path [CBM_SZ_1K ];
1505+ if (!cbm_safe_getenv ("CBM_SEM_STATE_DUMP" , state_path , sizeof (state_path ), NULL )) {
1506+ return ;
1507+ }
1508+ char path [CBM_SZ_1K ];
1509+ if (snprintf (path , sizeof (path ), "%s.%s" , state_path , suffix ) >= (int )sizeof (path )) {
1510+ return ;
1511+ }
1512+ int n = cbm_sem_corpus_token_count (corpus );
1513+ uint64_t h_vec = 1469598103934665603ULL ;
1514+ uint64_t h_idf = 1469598103934665603ULL ;
1515+ for (int i = 0 ; i < n ; i ++ ) {
1516+ const cbm_sem_vec_t * v = cbm_sem_corpus_ri_vec_at (corpus , i );
1517+ if (v ) {
1518+ const unsigned char * b = (const unsigned char * )v ;
1519+ for (size_t k = 0 ; k < sizeof (* v ); k ++ ) {
1520+ h_vec ^= b [k ];
1521+ h_vec *= 1099511628211ULL ;
1522+ }
1523+ }
1524+ uint32_t bits ;
1525+ float idf = cbm_sem_corpus_idf_at (corpus , i );
1526+ memcpy (& bits , & idf , sizeof (bits ));
1527+ h_idf ^= bits ;
1528+ h_idf *= 1099511628211ULL ;
1529+ }
1530+ FILE * sf = cbm_fopen (path , "w" );
1531+ if (sf ) {
1532+ fprintf (sf , "vec=%016llx idf=%016llx toks=%016llx tokens=%d\n" , (unsigned long long )h_vec ,
1533+ (unsigned long long )h_idf , (unsigned long long )g_sem_token_hash , n );
1534+ (void )fclose (sf );
1535+ }
1536+ }
1537+
13731538/* Phase 6a: score candidate pairs in parallel and collect deferred edges. */
13741539static void phase6a_score_candidates (cbm_sem_func_t * funcs , uint64_t * signatures , int * edge_counts ,
13751540 sem_bucket_t * * band_buckets , cbm_sem_config_t cfg ,
@@ -1415,11 +1580,13 @@ static cbm_sem_corpus_t *run_corpus_phase(cbm_gbuf_t *gbuf, char **all_tokens,
14151580 int func_count ) {
14161581 CBM_PROF_START (t_phase3a );
14171582 cbm_sem_corpus_t * corpus = cbm_sem_corpus_new ();
1583+ sem_tokens_hash (all_tokens , offsets , token_counts , func_count );
14181584 cbm_sem_corpus_add_docs_batch (corpus , all_tokens , offsets , token_counts , func_count );
14191585 CBM_PROF_END_N ("semantic_edges" , "3a_corpus_batch" , t_phase3a , func_count );
14201586
14211587 CBM_PROF_START (t_phase3b );
14221588 cbm_sem_corpus_finalize (corpus );
1589+ sem_corpus_dump (corpus , "corpus" );
14231590 CBM_PROF_END_N ("semantic_edges" , "3b_corpus_finalize_seq" , t_phase3b ,
14241591 cbm_sem_corpus_token_count (corpus ));
14251592
@@ -1449,6 +1616,7 @@ static int run_scoring_phase(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, uint64_t *
14491616 }
14501617
14511618 CBM_PROF_START (t_phase6a );
1619+ sem_state_dump (funcs , func_count );
14521620 phase6a_score_candidates (funcs , signatures , edge_counts , band_buckets , cfg , worker_bufs ,
14531621 func_count , worker_count );
14541622 CBM_PROF_END_N ("semantic_edges" , "6a_score_parallel" , t_phase6a , func_count );
@@ -1599,6 +1767,7 @@ static cbm_sem_corpus_t *run_token_phases_batched(cbm_gbuf_t *gbuf, cbm_sem_func
15991767 sem_batch_tokenize (node_ptrs , gbuf , first , count , worker_count , token_pools , token_counts ,
16001768 & tokens , & offsets );
16011769 if (tokens && offsets ) {
1770+ sem_tokens_hash (tokens , offsets , token_counts + first , count );
16021771 cbm_sem_corpus_add_docs_batch (corpus , tokens , offsets , token_counts + first , count );
16031772 }
16041773 sem_batch_free_tokens (tokens , offsets );
@@ -1609,6 +1778,7 @@ static cbm_sem_corpus_t *run_token_phases_batched(cbm_gbuf_t *gbuf, cbm_sem_func
16091778
16101779 CBM_PROF_START (t_phase3b );
16111780 cbm_sem_corpus_finalize (corpus );
1781+ sem_corpus_dump (corpus , "corpus" );
16121782 CBM_PROF_END_N ("semantic_edges" , "3b_corpus_finalize_seq" , t_phase3b ,
16131783 cbm_sem_corpus_token_count (corpus ));
16141784 CBM_PROF_START (t_phase3c );
0 commit comments