Skip to content

Commit 78a67e3

Browse files
authored
Break-glass merge PR #2607
2 parents a90076a + cfab5d7 commit 78a67e3

6 files changed

Lines changed: 244 additions & 5 deletions

File tree

‎src/graph_buffer/graph_buffer.c‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -835,6 +835,39 @@ void cbm_gbuf_set_next_id(cbm_gbuf_t *gb, int64_t next_id) {
835835
gb->next_id = next_id;
836836
}
837837

838+
void cbm_gbuf_raise_next_id(cbm_gbuf_t *gb, int64_t next_id) {
839+
if (gb && next_id > gb->next_id) {
840+
gb->next_id = next_id;
841+
}
842+
}
843+
844+
static int gb_id_cmp(const void *a, const void *b) {
845+
int64_t x = *(const int64_t *)a;
846+
int64_t y = *(const int64_t *)b;
847+
return (x > y) - (x < y);
848+
}
849+
850+
int cbm_gbuf_duplicate_edge_ids(const cbm_gbuf_t *gb) {
851+
if (!gb || gb->edges.count < 2) {
852+
return 0;
853+
}
854+
int n = gb->edges.count;
855+
int64_t *ids = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, (size_t)n * sizeof(*ids));
856+
if (!ids) {
857+
return CBM_NOT_FOUND;
858+
}
859+
for (int i = 0; i < n; i++) {
860+
ids[i] = gb->edges.items[i]->id;
861+
}
862+
qsort(ids, (size_t)n, sizeof(*ids), gb_id_cmp);
863+
int dups = 0;
864+
for (int i = 1; i < n; i++) {
865+
dups += ids[i] == ids[i - SKIP_ONE];
866+
}
867+
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, ids);
868+
return dups;
869+
}
870+
838871
/* ── Definition variants ─────────────────────────────────────────────
839872
* One name defined more than once under mutually exclusive conditions —
840873
* C-family #if/#else, Rust #[cfg], build-tagged files, a definition under an

‎src/graph_buffer/graph_buffer.h‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,9 +100,16 @@ int cbm_gbuf_node_count(const cbm_gbuf_t *gb);
100100
/* Get the next ID that would be assigned. Used to initialize shared atomic counters. */
101101
int64_t cbm_gbuf_next_id(const cbm_gbuf_t *gb);
102102

103-
/* Set the next ID counter. Used after merging worker gbufs to sync the main counter. */
103+
/* Set the next ID counter. Pins it: the delta preseed sets it to the DB's next id. */
104104
void cbm_gbuf_set_next_id(cbm_gbuf_t *gb, int64_t next_id);
105105

106+
/* Raise the next ID counter to at least next_id; never lowers it, so no id
107+
* already handed out comes back. Used to sync after merging worker gbufs. */
108+
void cbm_gbuf_raise_next_id(cbm_gbuf_t *gb, int64_t next_id);
109+
110+
/* Number of edges whose id an earlier edge already carries (0 = ids unique). */
111+
int cbm_gbuf_duplicate_edge_ids(const cbm_gbuf_t *gb);
112+
106113
/* Delete all nodes with a label. Cascade-deletes referencing edges. */
107114
int cbm_gbuf_delete_by_label(cbm_gbuf_t *gb, const char *label);
108115

‎src/pipeline/pipeline.c‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,9 @@ struct cbm_pipeline {
283283
* per pipeline so concurrent test/process activity cannot cross-trigger. */
284284
void (*before_publish_hook)(cbm_pipeline_t *, const char *, void *);
285285
void *before_publish_hook_ctx;
286+
/* Test-only: sees the complete graph buffer before the dump releases its indexes. */
287+
void (*before_dump_hook)(const cbm_gbuf_t *, void *);
288+
void *before_dump_hook_ctx;
286289
int (*rename_hook)(const char *, const char *, void *);
287290
void *rename_hook_ctx;
288291
};
@@ -1047,6 +1050,32 @@ void cbm_pipeline_set_before_publish_hook_for_tests(
10471050
}
10481051
}
10491052

1053+
void cbm_pipeline_set_before_dump_hook_for_tests(cbm_pipeline_t *p,
1054+
void (*hook)(const cbm_gbuf_t *, void *),
1055+
void *ctx) {
1056+
if (p) {
1057+
p->before_dump_hook = hook;
1058+
p->before_dump_hook_ctx = ctx;
1059+
}
1060+
}
1061+
1062+
static CBM_TLS void (*g_id_sync_hook)(cbm_gbuf_t *, void *);
1063+
static CBM_TLS void *g_id_sync_hook_ctx;
1064+
1065+
void cbm_pipeline_set_id_sync_hook_for_tests(void (*hook)(cbm_gbuf_t *, void *), void *ctx) {
1066+
g_id_sync_hook = hook;
1067+
g_id_sync_hook_ctx = ctx;
1068+
}
1069+
1070+
void cbm_pipeline_sync_next_id(cbm_gbuf_t *gb, _Atomic int64_t *shared_ids) {
1071+
if (g_id_sync_hook) {
1072+
g_id_sync_hook(gb, g_id_sync_hook_ctx);
1073+
}
1074+
/* Raise only: lowering it to the shared value handed the merged edges'
1075+
* ids out a second time. */
1076+
cbm_gbuf_raise_next_id(gb, atomic_load(shared_ids));
1077+
}
1078+
10501079
void cbm_pipeline_set_rename_hook_for_tests(cbm_pipeline_t *p,
10511080
int (*hook)(const char *, const char *, void *),
10521081
void *ctx) {
@@ -2291,7 +2320,7 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
22912320
cbm_pipeline_spill_close(ctx);
22922321
return rc != 0 ? rc : CBM_NOT_FOUND;
22932322
}
2294-
cbm_gbuf_set_next_id(p->gbuf, atomic_load(&shared_ids));
2323+
cbm_pipeline_sync_next_id(p->gbuf, &shared_ids);
22952324
/* extract -> registry handoff: return the extract phase's freed-but-retained
22962325
* allocator pages to the OS before registry_build allocates. On a 2x Linux
22972326
* index the extract peak holds ~13 GB of reclaimable pages (peak_mb 20.7 vs
@@ -2465,7 +2494,7 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
24652494
}
24662495
free(def_modules);
24672496
}
2468-
cbm_gbuf_set_next_id(p->gbuf, atomic_load(&shared_ids));
2497+
cbm_pipeline_sync_next_id(p->gbuf, &shared_ids);
24692498
cbm_pipeline_extract_infra_routes(ctx, p->gbuf, files, cache, file_count);
24702499
cbm_pipeline_process_infra_bindings(ctx, p->gbuf, files, cache, file_count);
24712500
for (int i = 0; i < file_count; i++) {
@@ -3415,6 +3444,9 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas
34153444
* release_gbuf_indexes(), which frees node_by_qn (graph_buffer.c), after
34163445
* which cbm_gbuf_node_count() returns 0. Reading these post-dump left
34173446
* committed_nodes at 0, so the #334 plausibility gate never fired. */
3447+
if (p->before_dump_hook) {
3448+
p->before_dump_hook(p->gbuf, p->before_dump_hook_ctx);
3449+
}
34183450
p->committed_nodes = cbm_gbuf_node_count(p->gbuf);
34193451
p->committed_edges = cbm_gbuf_edge_count(p->gbuf);
34203452
int rc = publish_generation_impl(&generation, p->frozen ? p : NULL);

‎src/pipeline/pipeline_incremental.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1742,7 +1742,7 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed
17421742
free_incremental_result_cache(cache, ci);
17431743
return rc;
17441744
}
1745-
cbm_gbuf_set_next_id(ctx->gbuf, atomic_load(&shared_ids));
1745+
cbm_pipeline_sync_next_id(ctx->gbuf, &shared_ids);
17461746

17471747
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
17481748
rc = cbm_build_registry_from_cache(ctx, changed_files, ci, cache);
@@ -1902,7 +1902,7 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed
19021902
}
19031903
cbm_log_info("pass.timing", "pass", "incr_resolve", "elapsed_ms",
19041904
itoa_buf((int)elapsed_ms(t)));
1905-
cbm_gbuf_set_next_id(ctx->gbuf, atomic_load(&shared_ids));
1905+
cbm_pipeline_sync_next_id(ctx->gbuf, &shared_ids);
19061906
free_incremental_result_cache(cache, ci);
19071907
if (legacy_cross_arena_live) {
19081908
free(all_defs); /* array returned by the existing collect helper */

‎src/pipeline/pipeline_internal.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1056,6 +1056,14 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
10561056
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
10571057
int worker_count);
10581058

1059+
/* Sync the main buffer's id counter after a parallel extract or resolve.
1060+
* Merged worker nodes keep their shared ids, and merged edges take fresh ids
1061+
* from the main counter after the highest of them, so the main counter can
1062+
* end above the shared one. */
1063+
void cbm_pipeline_sync_next_id(cbm_gbuf_t *gb, _Atomic int64_t *shared_ids);
1064+
/* Test seam: runs on the syncing thread right before each such sync. */
1065+
void cbm_pipeline_set_id_sync_hook_for_tests(void (*hook)(cbm_gbuf_t *, void *), void *ctx);
1066+
10591067
/* Phase 3B: Serial registry build from cached extraction results.
10601068
* Creates DEFINES, DEFINES_METHOD, and IMPORTS edges in ctx->gbuf.
10611069
* Registers callable symbols (Function/Method/Class) in ctx->registry. */
@@ -1440,6 +1448,11 @@ void cbm_pipeline_set_committed_counts(cbm_pipeline_t *p, int nodes, int edges);
14401448
* before the cancellation check + atomic replace. Not part of the public API. */
14411449
void cbm_pipeline_set_before_publish_hook_for_tests(
14421450
cbm_pipeline_t *p, void (*hook)(cbm_pipeline_t *, const char *, void *), void *ctx);
1451+
/* Test seam: invoked with the complete graph buffer just before the full
1452+
* route dumps it (the dump releases the buffer's indexes). */
1453+
void cbm_pipeline_set_before_dump_hook_for_tests(cbm_pipeline_t *p,
1454+
void (*hook)(const cbm_gbuf_t *, void *),
1455+
void *ctx);
14431456
void cbm_pipeline_set_rename_hook_for_tests(cbm_pipeline_t *p,
14441457
int (*hook)(const char *, const char *, void *),
14451458
void *ctx);

‎tests/test_parallel.c‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -958,6 +958,157 @@ TEST(gbuf_next_id_accessors) {
958958
PASS();
959959
}
960960

961+
TEST(gbuf_raise_next_id_never_lowers) {
962+
cbm_gbuf_t *gb = cbm_gbuf_new("proj", "/");
963+
cbm_gbuf_raise_next_id(gb, 100);
964+
ASSERT_EQ(cbm_gbuf_next_id(gb), 100);
965+
cbm_gbuf_raise_next_id(gb, 40);
966+
ASSERT_EQ(cbm_gbuf_next_id(gb), 100);
967+
int64_t id = cbm_gbuf_upsert_node(gb, "Function", "foo", "proj.foo", "f.go", 1, 5, "{}");
968+
ASSERT_EQ(id, 100);
969+
cbm_gbuf_free(gb);
970+
PASS();
971+
}
972+
973+
/* Edge ids in the main buffer stay unique through the parallel phases.
974+
* Merged worker nodes keep their shared ids, and merged edges take fresh ids
975+
* from the main counter after the highest of them, so the main counter can
976+
* end above the shared one (12 of 17 infra corpora, deckhouse by 229,233).
977+
* The sync after each parallel phase used to set the main counter to the
978+
* shared value anyway, and later passes reused the merged edges' ids. Where
979+
* the counter ends depends on worker timing, so the seam pins it: right before
980+
* each sync the hook takes ID_SYNC_PUSH edge ids from the main buffer. */
981+
enum { ID_FILES = 64, ID_SYNC_PUSH = 3 };
982+
983+
typedef struct {
984+
bool push;
985+
int syncs;
986+
} id_sync_probe_t;
987+
988+
static void id_sync_hook(cbm_gbuf_t *gb, void *ctx) {
989+
id_sync_probe_t *probe = ctx;
990+
probe->syncs++;
991+
const cbm_gbuf_node_t **fns = NULL;
992+
int n = 0;
993+
if (!probe->push || cbm_gbuf_find_by_label(gb, "Function", &fns, &n) != 0 || n < 2) {
994+
return;
995+
}
996+
char props[CBM_SZ_64];
997+
for (int i = 0; i < ID_SYNC_PUSH; i++) {
998+
snprintf(props, sizeof(props), "{\"id_sync_probe\":%d}", probe->syncs * ID_SYNC_PUSH + i);
999+
cbm_gbuf_insert_edge(gb, fns[0]->id, fns[1]->id, "CALLS", props);
1000+
}
1001+
}
1002+
1003+
typedef struct {
1004+
int edges;
1005+
int duplicates;
1006+
} edge_id_probe_t;
1007+
1008+
static void probe_edge_ids(const cbm_gbuf_t *gb, void *ctx) {
1009+
edge_id_probe_t *probe = ctx;
1010+
probe->edges = cbm_gbuf_edge_count(gb);
1011+
probe->duplicates = cbm_gbuf_duplicate_edge_ids(gb);
1012+
}
1013+
1014+
/* `count` Go files; each function calls the next file's. */
1015+
static bool write_id_fixture(const char *repo, int count) {
1016+
char path[512];
1017+
char source[256];
1018+
for (int i = 0; i < count; i++) {
1019+
snprintf(path, sizeof(path), "%s/f%02d.go", repo, i);
1020+
snprintf(source, sizeof(source),
1021+
"package main\n\nfunc F%02d() int {\n\treturn F%02d() + 1\n}\n", i,
1022+
(i + 1) % count);
1023+
if (th_write_file(path, source) != 0) {
1024+
return false;
1025+
}
1026+
}
1027+
return true;
1028+
}
1029+
1030+
TEST(parallel_full_index_keeps_edge_ids_unique) {
1031+
char repo[256];
1032+
snprintf(repo, sizeof(repo), "/tmp/cbm_par_ids_XXXXXX");
1033+
ASSERT_NOT_NULL(cbm_mkdtemp(repo));
1034+
/* More files than MIN_FILES_FOR_PARALLEL (50): the parallel route. */
1035+
if (!write_id_fixture(repo, ID_FILES)) {
1036+
th_rmtree(repo);
1037+
FAIL("failed to write the edge-id fixture");
1038+
}
1039+
char db_path[512];
1040+
snprintf(db_path, sizeof(db_path), "%s/ids.db", repo);
1041+
1042+
char *old_workers = getenv("CBM_WORKERS");
1043+
char *saved_workers = old_workers ? strdup(old_workers) : NULL;
1044+
cbm_setenv("CBM_WORKERS", "4", 1);
1045+
id_sync_probe_t syncs = {.push = true};
1046+
edge_id_probe_t probe = {0};
1047+
cbm_pipeline_set_id_sync_hook_for_tests(id_sync_hook, &syncs);
1048+
cbm_pipeline_t *p = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL);
1049+
int rc = CBM_NOT_FOUND;
1050+
if (p) {
1051+
cbm_pipeline_set_before_dump_hook_for_tests(p, probe_edge_ids, &probe);
1052+
rc = cbm_pipeline_run(p);
1053+
cbm_pipeline_free(p);
1054+
}
1055+
cbm_pipeline_set_id_sync_hook_for_tests(NULL, NULL);
1056+
if (saved_workers) {
1057+
cbm_setenv("CBM_WORKERS", saved_workers, 1);
1058+
free(saved_workers);
1059+
} else {
1060+
cbm_unsetenv("CBM_WORKERS");
1061+
}
1062+
th_rmtree(repo);
1063+
1064+
ASSERT_EQ(rc, 0);
1065+
/* Both full-route syncs (after extract, after resolve) go through the helper. */
1066+
ASSERT_EQ(syncs.syncs, 2);
1067+
ASSERT_GT(probe.edges, ID_FILES);
1068+
ASSERT_EQ(probe.duplicates, 0);
1069+
PASS();
1070+
}
1071+
1072+
/* The closure-repair route always runs the parallel phases; both of its
1073+
* syncs go through the same helper. */
1074+
TEST(parallel_closure_repair_syncs_ids_through_the_helper) {
1075+
char repo[256];
1076+
snprintf(repo, sizeof(repo), "/tmp/cbm_par_ids_XXXXXX");
1077+
ASSERT_NOT_NULL(cbm_mkdtemp(repo));
1078+
if (!write_id_fixture(repo, 3)) {
1079+
th_rmtree(repo);
1080+
FAIL("failed to write the edge-id fixture");
1081+
}
1082+
char db_path[512];
1083+
snprintf(db_path, sizeof(db_path), "%s/ids.db", repo);
1084+
cbm_pipeline_incremental_test_reset_faults();
1085+
cbm_pipeline_t *first = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL);
1086+
int first_rc = first ? cbm_pipeline_run(first) : CBM_NOT_FOUND;
1087+
cbm_pipeline_free(first);
1088+
1089+
/* A body-only edit (no new names) routes to closure repair. */
1090+
char path[512];
1091+
snprintf(path, sizeof(path), "%s/f01.go", repo);
1092+
int write_rc =
1093+
th_write_file(path, "package main\n\nfunc F01() int {\n\treturn F02() + 40 + 2\n}\n");
1094+
id_sync_probe_t syncs = {.push = false};
1095+
cbm_pipeline_set_id_sync_hook_for_tests(id_sync_hook, &syncs);
1096+
cbm_pipeline_incremental_test_reset_faults();
1097+
cbm_pipeline_t *second = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL);
1098+
int second_rc = second ? cbm_pipeline_run(second) : CBM_NOT_FOUND;
1099+
cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route();
1100+
cbm_pipeline_free(second);
1101+
cbm_pipeline_set_id_sync_hook_for_tests(NULL, NULL);
1102+
th_rmtree(repo);
1103+
1104+
ASSERT_EQ(first_rc, 0);
1105+
ASSERT_EQ(write_rc, 0);
1106+
ASSERT_EQ(second_rc, 0);
1107+
ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);
1108+
ASSERT_EQ(syncs.syncs, 2);
1109+
PASS();
1110+
}
1111+
9611112
/* ── Parallel-pipeline LSP-override regression ────────────────────── */
9621113
/* Pin the wiring fix that unified pass_calls.c (sequential) and
9631114
* pass_parallel.c (parallel) on cbm_pipeline_find_lsp_resolution +
@@ -4768,6 +4919,9 @@ SUITE(parallel) {
47684919
RUN_TEST(gbuf_merge_empty_src);
47694920
RUN_TEST(gbuf_merge_src_free_safe);
47704921
RUN_TEST(gbuf_next_id_accessors);
4922+
RUN_TEST(gbuf_raise_next_id_never_lowers);
4923+
RUN_TEST(parallel_full_index_keeps_edge_ids_unique);
4924+
RUN_TEST(parallel_closure_repair_syncs_ids_through_the_helper);
47714925

47724926
/* Parallel pipeline parity tests */
47734927
RUN_TEST(parallel_node_count);

0 commit comments

Comments
 (0)