Skip to content

Commit acb7eaa

Browse files
authored
Merge pull request DeusData#528 from win4r/fix/incremental-inbound-cross-file-edges
fix(pipeline): preserve inbound cross-file edges on incremental reindex
2 parents ff89931 + be3e038 commit acb7eaa

2 files changed

Lines changed: 278 additions & 2 deletions

File tree

‎src/pipeline/pipeline_incremental.c‎

Lines changed: 183 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,141 @@ static void free_mode_skipped(cbm_file_hash_t *ms, int count) {
296296
free(ms);
297297
}
298298

299+
/* ── Inbound cross-file edge preservation (incremental correctness) ──
300+
*
301+
* The purge step (cbm_gbuf_delete_by_file) removes a changed file's nodes,
302+
* and the cascade then drops every edge referencing them — INCLUDING inbound
303+
* edges whose source lives in an UNCHANGED file (e.g. StudyService.grade ->
304+
* SM2.review, or a Folder -> File containment edge). Because incremental only
305+
* re-parses the changed files, the resolution passes never regenerate those
306+
* inbound edges, so the graph silently loses cross-file CALLS / USAGE /
307+
* CONTAINS_FILE / INHERITS / ... edges on every edit and diverges from a
308+
* clean full reindex (which resolves every file).
309+
*
310+
* Fix: snapshot the inbound cross-file edges into changed files BEFORE the
311+
* purge, keyed by endpoint qualified_name (stable across re-parse), then
312+
* re-link them AFTER re-resolution + post-passes. Notes:
313+
* - Only edges whose target is in a changed file and whose source is NOT
314+
* are snapshotted; edges out of a changed file are regenerated when that
315+
* file is re-resolved.
316+
* - Edge types recomputed wholesale by post-passes (SIMILAR_TO,
317+
* SEMANTICALLY_RELATED) are skipped — re-linking a stale snapshot could
318+
* add edges a full reindex would not produce.
319+
* - cbm_gbuf_insert_edge dedups, so re-linking an edge the resolver already
320+
* recreated is a harmless no-op.
321+
* - A target whose qualified_name no longer exists (symbol deleted or
322+
* renamed by the edit) is dropped — matching full-reindex semantics. */
323+
324+
typedef struct {
325+
char *source_qn;
326+
char *target_qn;
327+
char *type;
328+
char *props;
329+
} cbm_saved_edge_t;
330+
331+
typedef struct {
332+
cbm_gbuf_t *gbuf;
333+
CBMHashTable *changed_paths; /* rel_path -> non-NULL sentinel (membership set) */
334+
cbm_saved_edge_t *items;
335+
int count;
336+
int cap;
337+
} cbm_edge_capture_t;
338+
339+
/* Edge types that must NOT be re-linked from the pre-purge snapshot, because a
340+
* full reindex (re)computes them via a pass whose result can differ from the
341+
* snapshot — restoring a stale copy could leave wrong properties or even an
342+
* edge a full reindex would not produce:
343+
* - SIMILAR_TO / SEMANTICALLY_RELATED: rebuilt wholesale by the incremental
344+
* post-passes (similarity / semantic_edges) over a drifting corpus.
345+
* - FILE_CHANGES_WITH (git-history coupling) and DATA_FLOWS (route data flow):
346+
* produced only by full-pipeline post-passes (githistory / route_nodes)
347+
* that do NOT run during incremental; they remain a known incremental
348+
* limitation rather than something to restore stale.
349+
* Every other edge type IS safe to re-link, by one of two routes that both
350+
* match a full reindex: edges re-emitted by the per-file resolution passes that
351+
* run incrementally (CALLS, USAGE, DEFINES, DEFINES_METHOD, INHERITS,
352+
* IMPLEMENTS) are deduped on re-link, while structural containment edges
353+
* (CONTAINS_FILE, CONTAINS_FOLDER) — which the full-only structure pass does
354+
* NOT regenerate incrementally — are preserved precisely by this snapshot. */
355+
static bool incr_edge_type_is_recomputed(const char *type) {
356+
return type && (strcmp(type, "SIMILAR_TO") == 0 || strcmp(type, "SEMANTICALLY_RELATED") == 0 ||
357+
strcmp(type, "FILE_CHANGES_WITH") == 0 || strcmp(type, "DATA_FLOWS") == 0);
358+
}
359+
360+
/* cbm_gbuf_foreach_edge visitor: snapshot inbound cross-file edges into
361+
* changed files so they survive the purge and can be re-linked afterward. */
362+
static void incr_capture_inbound_edge(const cbm_gbuf_edge_t *edge, void *userdata) {
363+
cbm_edge_capture_t *cap = (cbm_edge_capture_t *)userdata;
364+
if (incr_edge_type_is_recomputed(edge->type)) {
365+
return;
366+
}
367+
const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(cap->gbuf, edge->source_id);
368+
const cbm_gbuf_node_t *tgt = cbm_gbuf_find_by_id(cap->gbuf, edge->target_id);
369+
if (!src || !tgt || !src->qualified_name || !tgt->qualified_name || !src->file_path ||
370+
!tgt->file_path) {
371+
return;
372+
}
373+
/* Keep only edges that the purge would orphan permanently: target is in a
374+
* changed file (its node is deleted + re-created), source is NOT (its file
375+
* is never re-parsed, so the resolver won't regenerate the edge). */
376+
if (!cbm_ht_get(cap->changed_paths, tgt->file_path) ||
377+
cbm_ht_get(cap->changed_paths, src->file_path)) {
378+
return;
379+
}
380+
if (cap->count >= cap->cap) {
381+
int ncap = (cap->cap > 0) ? cap->cap * PAIR_LEN : CBM_SZ_64;
382+
cbm_saved_edge_t *tmp = realloc(cap->items, (size_t)ncap * sizeof(*tmp));
383+
if (!tmp) {
384+
cbm_log_warn("incremental.edge_snapshot_oom", "captured", itoa_buf(cap->count));
385+
return; /* best-effort: stop capturing, keep what we have */
386+
}
387+
cap->items = tmp;
388+
cap->cap = ncap;
389+
}
390+
cbm_saved_edge_t *s = &cap->items[cap->count];
391+
s->source_qn = strdup(src->qualified_name);
392+
s->target_qn = strdup(tgt->qualified_name);
393+
s->type = strdup(edge->type);
394+
s->props = strdup(edge->properties_json ? edge->properties_json : "{}");
395+
if (!s->source_qn || !s->target_qn || !s->type || !s->props) {
396+
free(s->source_qn);
397+
free(s->target_qn);
398+
free(s->type);
399+
free(s->props);
400+
return;
401+
}
402+
cap->count++;
403+
}
404+
405+
/* Re-link snapshotted inbound edges to the freshly re-created target nodes.
406+
* Returns the number of edges re-linked. */
407+
static int incr_restore_inbound_edges(cbm_gbuf_t *gbuf, cbm_edge_capture_t *cap) {
408+
int restored = 0;
409+
for (int i = 0; i < cap->count; i++) {
410+
cbm_saved_edge_t *s = &cap->items[i];
411+
const cbm_gbuf_node_t *src = cbm_gbuf_find_by_qn(gbuf, s->source_qn);
412+
const cbm_gbuf_node_t *tgt = cbm_gbuf_find_by_qn(gbuf, s->target_qn);
413+
if (src && tgt) {
414+
cbm_gbuf_insert_edge(gbuf, src->id, tgt->id, s->type, s->props);
415+
restored++;
416+
}
417+
}
418+
return restored;
419+
}
420+
421+
static void incr_free_edge_capture(cbm_edge_capture_t *cap) {
422+
for (int i = 0; i < cap->count; i++) {
423+
free(cap->items[i].source_qn);
424+
free(cap->items[i].target_qn);
425+
free(cap->items[i].type);
426+
free(cap->items[i].props);
427+
}
428+
free(cap->items);
429+
cap->items = NULL;
430+
cap->count = 0;
431+
cap->cap = 0;
432+
}
433+
299434
/* ── Persist file hashes ─────────────────────────────────────────── */
300435

301436
/* Persist file hash rows for the current discovery and any mode-skipped
@@ -367,10 +502,27 @@ static void persist_hashes(cbm_store_t *store, const char *project, cbm_file_inf
367502

368503
/* ── Registry seed visitor ────────────────────────────────────────── */
369504

370-
/* Callback for cbm_gbuf_foreach_node: add each node to the registry
371-
* so the resolver can find cross-file symbols during incremental. */
505+
/* Labels the full-index definition pass seeds into the registry
506+
* (pass_definitions.c — KEEP IN SYNC). Incremental re-resolution must see the
507+
* SAME symbol set, or it diverges from a clean full reindex: seeding extra
508+
* container nodes (File / Module / Folder / ...) lets a type usage like `Word`
509+
* resolve to the same-named Module node instead of the Class node. Only
510+
* callable / declared symbols belong in the registry. */
511+
static bool incr_label_is_registry_symbol(const char *label) {
512+
return label && (strcmp(label, "Function") == 0 || strcmp(label, "Method") == 0 ||
513+
strcmp(label, "Class") == 0 || strcmp(label, "Interface") == 0 ||
514+
strcmp(label, "Variable") == 0 || strcmp(label, "Field") == 0);
515+
}
516+
517+
/* Callback for cbm_gbuf_foreach_node: seed the registry with the existing
518+
* project's definition symbols so the resolver can match cross-file symbols
519+
* during incremental. Mirrors the full-index registry contents exactly so an
520+
* incremental re-resolve picks the same nodes a full reindex would. */
372521
static void registry_visitor(const cbm_gbuf_node_t *node, void *userdata) {
373522
cbm_registry_t *r = (cbm_registry_t *)userdata;
523+
if (!incr_label_is_registry_symbol(node->label)) {
524+
return;
525+
}
374526
cbm_registry_add(r, node->name, node->qualified_name, node->label);
375527
}
376528

@@ -612,6 +764,25 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
612764

613765
cbm_store_close(store);
614766

767+
/* Snapshot inbound cross-file edges into changed files BEFORE purging, so
768+
* the cascade delete doesn't permanently drop edges whose source lives in
769+
* an unchanged (never-re-parsed) file. Re-linked after re-resolution. */
770+
cbm_edge_capture_t edge_cap = {0};
771+
edge_cap.gbuf = existing;
772+
{
773+
CBMHashTable *changed_paths = cbm_ht_create(ci > 0 ? (size_t)ci * PAIR_LEN : CBM_SZ_64);
774+
for (int i = 0; i < ci; i++) {
775+
cbm_ht_set(changed_paths, changed_files[i].rel_path, &changed_files[i]);
776+
}
777+
edge_cap.changed_paths = changed_paths;
778+
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
779+
cbm_gbuf_foreach_edge(existing, incr_capture_inbound_edge, &edge_cap);
780+
edge_cap.changed_paths = NULL;
781+
cbm_ht_free(changed_paths); /* keys borrowed from changed_files; not freed here */
782+
}
783+
cbm_log_info("incremental.edge_snapshot", "captured", itoa_buf(edge_cap.count), "elapsed_ms",
784+
itoa_buf((int)elapsed_ms(t)));
785+
615786
/* Step 2: Purge stale nodes */
616787
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
617788
for (int i = 0; i < ci; i++) {
@@ -660,6 +831,16 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
660831
cbm_registry_free(registry);
661832
cbm_path_alias_collection_free(path_aliases);
662833

834+
/* Re-link inbound cross-file edges that the purge orphaned. Runs after
835+
* re-resolution AND post-passes so the freshly re-created target nodes
836+
* exist and nothing downstream clobbers the restored edges; insert_edge
837+
* dedups, so any edge the resolver already recreated is a no-op. */
838+
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
839+
int relinked = incr_restore_inbound_edges(existing, &edge_cap);
840+
cbm_log_info("incremental.edge_relink", "relinked", itoa_buf(relinked), "captured",
841+
itoa_buf(edge_cap.count), "elapsed_ms", itoa_buf((int)elapsed_ms(t)));
842+
incr_free_edge_capture(&edge_cap);
843+
663844
/* Step 7: Dump to disk (preserves mode-skipped hash rows so the next
664845
* reindex can correctly classify those files instead of seeing them
665846
* as never-existed; also exports a fast-mode artifact when one is

‎tests/test_pipeline.c‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -743,6 +743,100 @@ TEST(pipeline_calls_resolution) {
743743
PASS();
744744
}
745745

746+
/* True iff a CALLS edge exists from a node named src_name to a node named
747+
* tgt_name. Used to assert cross-file call resolution survives a reindex. */
748+
static bool cross_file_call_exists(cbm_store_t *s, const char *project, const char *src_name,
749+
const char *tgt_name) {
750+
cbm_node_t *srcs = NULL;
751+
cbm_node_t *tgts = NULL;
752+
int sc = 0;
753+
int tc = 0;
754+
cbm_store_find_nodes_by_name(s, project, src_name, &srcs, &sc);
755+
cbm_store_find_nodes_by_name(s, project, tgt_name, &tgts, &tc);
756+
bool found = false;
757+
for (int i = 0; i < sc && !found; i++) {
758+
cbm_edge_t *edges = NULL;
759+
int ec = 0;
760+
cbm_store_find_edges_by_source_type(s, srcs[i].id, "CALLS", &edges, &ec);
761+
for (int j = 0; j < ec && !found; j++) {
762+
for (int k = 0; k < tc; k++) {
763+
if (edges[j].target_id == tgts[k].id) {
764+
found = true;
765+
break;
766+
}
767+
}
768+
}
769+
if (edges) {
770+
cbm_store_free_edges(edges, ec);
771+
}
772+
}
773+
if (srcs) {
774+
cbm_store_free_nodes(srcs, sc);
775+
}
776+
if (tgts) {
777+
cbm_store_free_nodes(tgts, tc);
778+
}
779+
return found;
780+
}
781+
782+
/* Regression: incremental re-index of an edited file must NOT drop inbound
783+
* cross-file CALLS edges whose source lives in an UNCHANGED file.
784+
*
785+
* Repro: Serve() (pkg/service.go) CALLS Help() (pkg/util/helper.go). Editing
786+
* helper.go purges Help's node; before the fix the cascade deleted the
787+
* Serve->Help edge and never regenerated it (helper.go's callers are not
788+
* re-parsed), so the edge silently vanished on every edit and the incremental
789+
* graph diverged from a full reindex. */
790+
TEST(pipeline_incremental_preserves_cross_file_calls) {
791+
if (setup_test_repo() != 0) {
792+
FAIL("failed to create temp dir");
793+
}
794+
795+
char db_path[512];
796+
snprintf(db_path, sizeof(db_path), "%s/test_incr_calls.db", g_tmpdir);
797+
798+
/* 1. Full index. */
799+
cbm_pipeline_t *p1 = cbm_pipeline_new(g_tmpdir, db_path, CBM_MODE_FULL);
800+
ASSERT_NOT_NULL(p1);
801+
ASSERT_EQ(cbm_pipeline_run(p1), 0);
802+
const char *project = cbm_pipeline_project_name(p1);
803+
804+
cbm_store_t *s1 = cbm_store_open_path(db_path);
805+
ASSERT_NOT_NULL(s1);
806+
int calls_before = cbm_store_count_edges_by_type(s1, project, "CALLS");
807+
ASSERT_GTE(calls_before, 2); /* main->Serve and Serve->Help at minimum */
808+
ASSERT_TRUE(cross_file_call_exists(s1, project, "Serve", "Help"));
809+
cbm_store_close(s1);
810+
cbm_pipeline_free(p1);
811+
812+
/* 2. Edit the callee's file so the incremental classifier marks it changed
813+
* (mtime+size differ). Help's symbol + qualified name are unchanged. */
814+
char helper[512];
815+
snprintf(helper, sizeof(helper), "%s/pkg/util/helper.go", g_tmpdir);
816+
ASSERT_EQ(th_append_file(helper, "\n// incremental regression marker\n"), 0);
817+
818+
/* 3. Re-run on the SAME db_path → auto-routes to incremental re-index. */
819+
cbm_pipeline_t *p2 = cbm_pipeline_new(g_tmpdir, db_path, CBM_MODE_FULL);
820+
ASSERT_NOT_NULL(p2);
821+
ASSERT_EQ(cbm_pipeline_run(p2), 0);
822+
823+
/* 4. The inbound cross-file CALLS edge must survive and the total CALLS
824+
* count must not regress. (Before the fix: Serve->Help is dropped.)
825+
* NOTE: query with p2's project name — p1 (and the `project` pointer it
826+
* owned) was freed above; p1 and p2 derive the same name from g_tmpdir. */
827+
const char *project2 = cbm_pipeline_project_name(p2);
828+
cbm_store_t *s2 = cbm_store_open_path(db_path);
829+
ASSERT_NOT_NULL(s2);
830+
int calls_after = cbm_store_count_edges_by_type(s2, project2, "CALLS");
831+
ASSERT_EQ(calls_after, calls_before);
832+
ASSERT_TRUE(cross_file_call_exists(s2, project2, "Serve", "Help"));
833+
cbm_store_close(s2);
834+
cbm_pipeline_free(p2);
835+
836+
teardown_test_repo();
837+
PASS();
838+
}
839+
746840
/* ── Git history pass tests ─────────────────────────────────────── */
747841

748842
TEST(githistory_is_trackable) {
@@ -5968,6 +6062,7 @@ SUITE(pipeline) {
59686062
RUN_TEST(pipeline_complexity_transitive_loop_depth);
59696063
/* Calls pass */
59706064
RUN_TEST(pipeline_calls_resolution);
6065+
RUN_TEST(pipeline_incremental_preserves_cross_file_calls);
59716066
/* Git history pass */
59726067
RUN_TEST(githistory_is_trackable);
59736068
RUN_TEST(githistory_compute_coupling);

0 commit comments

Comments
 (0)