From d3b8d258bec7b10c739e26fa71c436a3f903df0a Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 12 Oct 2026 19:45:10 +0200 Subject: [PATCH] feat(index): doc_candidates setting and index_repository parameter The doc<->code candidate scoring (#2574, doc_section_worker in semantic_edges) scores every documentation section against the functions that share a key term with it, key terms being those held by at most funcs/20 functions. Its cost therefore grows with sections x functions. Measured on 11 corpora (10-09 main vs 4d9592b5, semantic_edges): under a second up to ~100k functions, +12.6 s on java (325k), +14.6 s on csharp (451k), +174 s on the Linux kernel (784k functions, 28k sections), almost all of the kernel's regression against v0.11.0. A new boolean setting, doc_candidates (default true), and an optional index_repository argument of the same name switch the scoring off: no candidate rows are stored and nodes and edges are unchanged. The argument wins over the setting. The index handler resolves the value once and passes it to a daemon job or supervised worker as an argument, the same way as the resource policy, so every route sees one decision; the session auto-index reads the setting. With the switch off, a delta-route reindex replaces the candidate rows it would otherwise keep, so turning it off leaves no stale candidates. The default output is unchanged. Tests (each fails on its assertion when the switch has no effect): - pipeline_doc_candidates_off_stores_none_and_keeps_graph: no rows, same node and edge counts as the default run, which has rows; - pipeline_doc_candidates_off_clears_rows_on_delta: a closure-repair reindex with the switch off removes the earlier rows; - tool_index_repository_doc_candidates_switch: argument false -> none; setting default -> rows; setting false -> none; argument true over setting false -> rows. Proof, release-built binaries on macOS arm64 (M5 Pro 18c/48 GB): - django, main 4d9592b5 vs this change with defaults: nodes (63,441), edges (307,584) and doc_link_candidates rows (11,304) hash-identical; - django with doc_candidates false: nodes and edges hash-identical, no candidate rows; - Linux kernel with doc_candidates false: semantic_edges 17.9 s (191.9 s on main, 17.6 s before #2574), whole index 210.9 s (v0.11.0: ~234 s), no candidate rows. Signed-off-by: Martin Vogel --- README.md | 1 + docs/CONFIGURATION.md | 1 + src/cli/cli.h | 1 + src/cli/runtime_settings.c | 4 + src/mcp/mcp.c | 41 ++++++-- src/pipeline/pass_semantic_edges.c | 6 ++ src/pipeline/pipeline.c | 12 +++ src/pipeline/pipeline.h | 7 ++ src/pipeline/pipeline_incremental.c | 4 +- tests/test_mcp.c | 152 +++++++++++++++++++++++++++ tests/test_pipeline.c | 154 ++++++++++++++++++++++++++-- 11 files changed, 367 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 1ef376d9a0..795849a206 100644 --- a/README.md +++ b/README.md @@ -793,6 +793,7 @@ codebase-memory-mcp config list # show all settings codebase-memory-mcp config set auto_index true # auto-index on session start codebase-memory-mcp config set auto_index_limit 50000 # max files for auto-index codebase-memory-mcp config set auto_watch false # don't register background git watcher (default: true) +codebase-memory-mcp config set doc_candidates false # skip doc<->code candidate scoring on huge repos (default: true) codebase-memory-mcp config set watcher_enabled false # stop the watcher thread entirely (default: true) codebase-memory-mcp config set watch_non_git true # also poll non-git project roots (default: false) codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 0ee1f41eb1..b26378e95d 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -88,6 +88,7 @@ Current keys: | `auto_index` | `false` | Automatically index new projects when an MCP session starts. | | `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. | | `auto_watch` | `true` | Register the session's project with the background git watcher on connect. Set `false` to keep a session from registering its project (the watcher still runs for other projects). | +| `doc_candidates` | `true` | Score documentation sections against code to store doc↔code candidates (the possible targets of a section, and the sections that may describe a function or file, each with a judged probability). The scoring grows with sections × functions: under a second up to ~100k functions, ~15 s at ~400k, ~3 minutes on the Linux kernel. Set `false` to skip it; no candidates are stored and the rest of the graph is unchanged. `index_repository` also takes `doc_candidates` for a single index, which wins over this setting. Applies from the next index. | | `watcher_enabled` | `true` | Master switch for the background watcher subsystem. Set `false` to stop the watcher from starting at all — no poll thread and no project registration. Reindex manually with `index_repository` when disabled. Only an open MCP session's own project is ever watched; `index_status` reports the current state in its `watch` object. | | `watch_non_git` | `false` | Also poll project roots that are **not git repositories**. By default the watcher only follows git projects, so a project indexed from a plain directory is never refreshed after its first index — reindex it manually with `index_repository`. Set `true` to poll such roots on the same adaptive cadence with a file-tree scan: the indexer's own discovery walk (same skip lists, `.gitignore` and `.cbmignore` rules) hashed over each file's path, size and mtime. Any change reindexes once; paths the indexer skips (including cbm's own `.codebase-memory/` output) never trigger. The first poll after the daemon starts reindexes each such project once, since nothing records which tree state the index holds. The scan walks the whole tree every poll, so it costs more than git polling on very large trees. Read once when the daemon starts, like `watcher_enabled`. | | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | diff --git a/src/cli/cli.h b/src/cli/cli.h index ed7524604e..a0274502ab 100644 --- a/src/cli/cli.h +++ b/src/cli/cli.h @@ -452,6 +452,7 @@ bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t #define CBM_CONFIG_AUTO_INDEX "auto_index" #define CBM_CONFIG_AUTO_INDEX_LIMIT "auto_index_limit" #define CBM_CONFIG_AUTO_WATCH "auto_watch" +#define CBM_CONFIG_DOC_CANDIDATES "doc_candidates" #define CBM_CONFIG_UI_LANG "ui-lang" #define CBM_CONFIG_WATCHER_ENABLED "watcher_enabled" #define CBM_CONFIG_WATCH_NON_GIT "watch_non_git" diff --git a/src/cli/runtime_settings.c b/src/cli/runtime_settings.c index d8ce46c2a3..ff98e55574 100644 --- a/src/cli/runtime_settings.c +++ b/src/cli/runtime_settings.c @@ -46,6 +46,10 @@ static const setting_t settings[] = { CFG("auto_watch", "Automatic watching", "Register a background Git watcher when an MCP session connects.", "boolean", "true", 0, 0, NULL, "src/mcp/mcp.c", "next-session"), + CFG("doc_candidates", "Doc-code candidates", + "Score documentation sections against code for doc-code candidates. Costly on very large " + "repositories; off stores none.", + "boolean", "true", 0, 0, NULL, "src/mcp/mcp.c", "next-session"), CFG("watcher_enabled", "Background watcher", "Enable the daemon's background watcher thread.", "boolean", "true", 0, 0, NULL, "src/daemon/application.c", "restart"), {"ui-lang", "Interface language", "Language preference used by subsequent HTTP requests.", diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index e89120eee4..412e922694 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -548,6 +548,8 @@ static const tool_def_t TOOLS[] = { "\"Name override; Non-ASCII bytes are encoded; unsafe characters normalized.\"}," "\"persistence\":{\"type\":\"boolean\",\"default\":false,\"description\":" "\"Write .codebase-memory/graph.db.zst.\"}," + "\"doc_candidates\":{\"type\":\"boolean\",\"description\":" + "\"Doc-code candidates; default: the doc_candidates setting (on).\"}," "\"async\":{\"type\":\"boolean\",\"default\":false,\"description\":" "\"Start the index in the background and return immediately; it keeps running if this " "call is cancelled or times out. Not with status or cross-repo-intelligence. Refused for a " @@ -11738,14 +11740,32 @@ static bool resolve_session_repo_path(cbm_mcp_server_t *srv, char **repo_path) { return true; } +/* doc_candidates for one index: an explicit boolean argument wins, else the + * doc_candidates setting (default on). The parent resolves it once and passes + * the decided value on, so a daemon job or supervised worker gets it as an + * argument instead of reading a config of its own. */ +static bool index_doc_candidates_enabled(cbm_mcp_server_t *srv, const char *args) { + yyjson_doc *doc = args ? yyjson_read(args, strlen(args), 0) : NULL; + yyjson_val *val = doc ? yyjson_obj_get(yyjson_doc_get_root(doc), "doc_candidates") : NULL; + bool explicit_value = val && yyjson_is_bool(val); + bool enabled = explicit_value ? yyjson_get_bool(val) : true; + yyjson_doc_free(doc); + if (!explicit_value && srv && srv->config) { + enabled = cbm_config_get_bool(srv->config, CBM_CONFIG_DOC_CANDIDATES, true); + } + return enabled; +} + /* Preserve every index option while replacing all caller-supplied repo_path * keys with the one canonical path that was actually authorized. A non-empty * project_name likewise replaces any caller "name" (#2134: the project key the * parent resolved for this root travels to the daemon job / worker verbatim, - * so lease key and written index cannot diverge). */ + * so lease key and written index cannot diverge). doc_candidates is replaced + * by the value the parent resolved. */ static char *index_args_with_repo_path(const char *args, const char *canonical_repo_path, const char *project_name, - const cbm_index_resource_policy_t *policy) { + const cbm_index_resource_policy_t *policy, + bool doc_candidates) { if (!args || !canonical_repo_path || !policy) { return NULL; } @@ -11783,9 +11803,13 @@ static char *index_args_with_repo_path(const char *args, const char *canonical_r while (has_name && yyjson_mut_obj_get(copy_root, "name")) { (void)yyjson_mut_obj_remove_key(copy_root, "name"); } + while (yyjson_mut_obj_get(copy_root, "doc_candidates")) { + (void)yyjson_mut_obj_remove_key(copy_root, "doc_candidates"); + } if ((has_name && !yyjson_mut_obj_add_strcpy(copy, copy_root, "name", project_name)) || !yyjson_mut_obj_add_strcpy(copy, copy_root, "repo_path", canonical_repo_path) || - !cbm_mcp_index_policy_add_to_args(copy, copy_root, policy)) { + !cbm_mcp_index_policy_add_to_args(copy, copy_root, policy) || + !yyjson_mut_obj_add_bool(copy, copy_root, "doc_candidates", doc_candidates)) { yyjson_mut_doc_free(copy); return NULL; } @@ -12151,12 +12175,13 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { index_args_free(repo_path, mode_str, name_override); return cbm_mcp_text_result(policy_error, true); } + bool doc_candidates = index_doc_candidates_enabled(srv, args); /* A daemon session delegates the one physical write to its shared job * registry only after path canonicalization and workspace authorization. */ if (srv->index_executor) { - char *worker_args = - index_args_with_repo_path(args, repo_path, name_override, &resource_policy); + char *worker_args = index_args_with_repo_path(args, repo_path, name_override, + &resource_policy, doc_candidates); char *coordinated = worker_args ? srv->index_executor(srv->index_executor_context, repo_path, worker_args, async_mode) : NULL; @@ -12184,8 +12209,8 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { * installs the same guard before running the in-process pipeline. A marked * host fails closed if preparation or worker startup cannot complete. */ if (cbm_index_supervisor_should_wrap()) { - char *worker_args = - index_args_with_repo_path(args, repo_path, name_override, &resource_policy); + char *worker_args = index_args_with_repo_path(args, repo_path, name_override, + &resource_policy, doc_candidates); if (!worker_args) { free(mutation_project); index_args_free(repo_path, mode_str, name_override); @@ -12249,6 +12274,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { free(name_override); cbm_pipeline_set_persistence(p, persistence); cbm_pipeline_set_resource_policy(p, &resource_policy); + cbm_pipeline_set_doc_candidates_enabled(p, doc_candidates); char *project_name = heap_strdup(cbm_pipeline_project_name(p)); @@ -19957,6 +19983,7 @@ static void *autoindex_thread(void *arg) { cbm_log_warn("autoindex.err", "msg", "pipeline_create_failed"); return NULL; } + cbm_pipeline_set_doc_candidates_enabled(p, index_doc_candidates_enabled(srv, NULL)); /* Block until any concurrent pipeline finishes */ cbm_pipeline_lock(); diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c index e5822d8096..bfde1d6bce 100644 --- a/src/pipeline/pass_semantic_edges.c +++ b/src/pipeline/pass_semantic_edges.c @@ -3029,6 +3029,12 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t * if (!corpus || func_count <= 0) { return; } + /* doc_candidates off: no scoring and no rows (the scoring grows with + * sections x functions; on the Linux kernel it is ~174 s of the index). */ + if (!cbm_pipeline_doc_candidates_enabled(ctx->pipeline)) { + cbm_log_info("pass.semantic.doc_candidates", "skipped", "doc_candidates_off"); + return; + } const cbm_gbuf_node_t **found = NULL; int nsec = 0; if (cbm_gbuf_find_by_label(gbuf, "Section", &found, &nsec) != 0 || nsec == 0) { diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index dd93d63ded..a8aa7dc4fc 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -278,6 +278,8 @@ struct cbm_pipeline { * cbm_pipeline_free). NULL when the pass found none or did not run. */ cbm_doc_candidate_t *doc_candidates; int doc_candidate_count; + /* doc_candidates setting off (calloc'd false = on, the default). */ + bool doc_candidates_off; /* Deterministic test-only seam at the final publication boundary. Kept * per pipeline so concurrent test/process activity cannot cross-trigger. */ @@ -871,6 +873,16 @@ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled) { } } +void cbm_pipeline_set_doc_candidates_enabled(cbm_pipeline_t *p, bool enabled) { + if (p) { + p->doc_candidates_off = !enabled; + } +} + +bool cbm_pipeline_doc_candidates_enabled(const cbm_pipeline_t *p) { + return !p || !p->doc_candidates_off; +} + void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, const cbm_index_resource_policy_t *policy) { if (p && policy) { diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 5ce3642f3b..fa7c88def0 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -56,6 +56,13 @@ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled); /* Apply a validated discovery resource policy. The value is copied. */ void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, const cbm_index_resource_policy_t *policy); +/* Doc -> code candidates (the doc_candidates setting; default on). Off skips + * scoring documentation sections against code and leaves the project with no + * doc_link_candidates rows; nodes and edges are unchanged. The scoring grows + * with sections x functions, which is the cost on very large repositories. */ +void cbm_pipeline_set_doc_candidates_enabled(cbm_pipeline_t *p, bool enabled); +bool cbm_pipeline_doc_candidates_enabled(const cbm_pipeline_t *p); + /* Copy the exact discovery violation from the most recent run. */ void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p, cbm_index_resource_violation_t *violation); diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 330d36d898..2c3fb15483 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -3172,7 +3172,9 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char .doc_link_rows = doc_rows, .doc_link_row_count = doc_row_count, .doc_links_failed = doc_failed, - .doc_candidates_in_place = true, /* the clone keeps the previous rows */ + /* The clone keeps the previous rows; with doc_candidates off they + * are replaced by none, so turning it off leaves no stale rows. */ + .doc_candidates_in_place = cbm_pipeline_doc_candidates_enabled(p), }; cbm_store_close(staging); staging = NULL; diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 4c2aeaad81..48cb5556f4 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -15257,6 +15257,157 @@ TEST(tool_index_repository_resolves_root_path_from_project_name_issue1211) { PASS(); } +/* A Go config family per subject (parse / load / validate) and a README + * section about the Mail one: enough for doc -> code candidates. */ +static void doccand_fixture(const char *repo) { + static const char *const subjects[] = {"User", "Server", "Cache", "Mail"}; + for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) { + const char *s = subjects[i]; + char path[512]; + snprintf(path, sizeof(path), "%s/%s_config.go", repo, s); + char body[2048]; + snprintf(body, sizeof(body), + "package main\n\n" + "// Parse%sConfig reads the %s config file and returns the parsed %s config.\n" + "func Parse%sConfig(path string) (*%sConfig, error) {\n" + "\treturn Load%sConfig(path)\n}\n\n" + "// Load%sConfig loads the %s config from disk and validates it.\n" + "func Load%sConfig(path string) (*%sConfig, error) {\n" + "\tcfg := &%sConfig{}\n\tValidate%sConfig(cfg)\n\treturn cfg, nil\n}\n\n" + "// Validate%sConfig checks the %s config for missing fields.\n" + "func Validate%sConfig(cfg *%sConfig) bool {\n\treturn cfg != nil\n}\n\n" + "type %sConfig struct{ Name string }\n", + s, s, s, s, s, s, s, s, s, s, s, s, s, s, s, s, s); + th_write_file(path, body); + } + char readme[512]; + snprintf(readme, sizeof(readme), "%s/README.md", repo); + th_write_file(readme, "# Mail configuration\n\n" + "The mail config file is parsed by ParseMailConfig, which loads the mail " + "config from disk and validates the mail config fields before anything " + "reads them. Missing fields fail the validation, and the parsed mail " + "config can be written back to disk afterwards.\n"); +} + +/* The one project DB in a private cache (not _config.db or activity.db), so + * the test does not depend on how the server canonicalized the repo path. */ +static bool doccand_project_db(const char *cache, char *out, size_t out_size) { + cbm_dir_t *d = cbm_opendir(cache); + if (!d) { + return false; + } + bool found = false; + cbm_dirent_t *e; + while (!found && (e = cbm_readdir(d)) != NULL) { + size_t len = strlen(e->name); + if (!e->is_dir && len > 3 && strcmp(e->name + len - 3, ".db") == 0 && + strcmp(e->name, "_config.db") != 0 && strcmp(e->name, "activity.db") != 0) { + snprintf(out, out_size, "%s/%s", cache, e->name); + found = true; + } + } + cbm_closedir(d); + return found; +} + +/* Remove the project DB (and its WAL/SHM) so the next index is a full one. */ +static void doccand_clear(const char *cache) { + char path[CBM_SZ_4K]; + if (!doccand_project_db(cache, path, sizeof(path))) { + return; + } + char side[CBM_SZ_4K + 8]; + snprintf(side, sizeof(side), "%s-wal", path); + cbm_unlink(side); + snprintf(side, sizeof(side), "%s-shm", path); + cbm_unlink(side); + cbm_unlink(path); +} + +/* The project's doc_link_candidates rows (0 without the table); -1 on an + * error. */ +static int doccand_rows(const char *cache) { + char path[CBM_SZ_4K]; + if (!doccand_project_db(cache, path, sizeof(path))) { + return -1; + } + cbm_store_t *s = cbm_store_open_path(path); + if (!s) { + return -1; + } + sqlite3 *db = cbm_store_get_db(s); + sqlite3_stmt *st = NULL; + int n = -1; + if (sqlite3_prepare_v2(db, + "SELECT count(*) FROM sqlite_master WHERE type='table' AND " + "name='doc_link_candidates'", + -1, &st, NULL) == SQLITE_OK) { + bool has_table = sqlite3_step(st) == SQLITE_ROW && sqlite3_column_int(st, 0) > 0; + sqlite3_finalize(st); + st = NULL; + n = 0; + if (has_table && sqlite3_prepare_v2(db, "SELECT count(*) FROM doc_link_candidates", -1, + &st, NULL) == SQLITE_OK) { + n = sqlite3_step(st) == SQLITE_ROW ? sqlite3_column_int(st, 0) : -1; + sqlite3_finalize(st); + } + } + cbm_store_close(s); + return n; +} + +/* One fresh full index (the project DB removed first, so no incremental route + * can keep earlier rows); returns the candidate rows, -1 on an index error. */ +static int doccand_index(cbm_mcp_server_t *srv, const char *cache, const char *repo, + const char *extra) { + doccand_clear(cache); + char args[1024]; + snprintf(args, sizeof(args), "{\"repo_path\":\"%s\"%s}", repo, extra); + char *resp = cbm_mcp_handle_tool(srv, "index_repository", args); + bool ok = resp && response_contains_json_fragment(resp, "\"status\":\"indexed\""); + free(resp); + return ok ? doccand_rows(cache) : -1; +} + +/* index_repository doc_candidates: an explicit false stores no candidate + * rows; without the argument the doc_candidates setting decides (default on, + * off stores none); an explicit true wins over the setting. */ +TEST(tool_index_repository_doc_candidates_switch) { + char repo[256]; + snprintf(repo, sizeof(repo), "/tmp/cbm-doccand-repo-XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(repo)); + char cache[256]; + snprintf(cache, sizeof(cache), "/tmp/cbm-doccand-cache-XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(cache)); + const char *saved = getenv("CBM_CACHE_DIR"); + char *saved_copy = saved ? strdup(saved) : NULL; + cbm_setenv("CBM_CACHE_DIR", cache, 1); + doccand_fixture(repo); + cbm_config_t *cfg = cbm_config_open(cache); + ASSERT_NOT_NULL(cfg); + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_mcp_server_set_config(srv, cfg); + + int explicit_off = doccand_index(srv, cache, repo, ",\"doc_candidates\":false"); + int setting_default = doccand_index(srv, cache, repo, ""); + cbm_config_set(cfg, CBM_CONFIG_DOC_CANDIDATES, "false"); + int setting_off = doccand_index(srv, cache, repo, ""); + int explicit_on = doccand_index(srv, cache, repo, ",\"doc_candidates\":true"); + + cbm_mcp_server_free(srv); + cbm_config_close(cfg); + restore_cache_dir(saved_copy); + free(saved_copy); + th_rmtree(cache); + th_rmtree(repo); + ASSERT_EQ(explicit_off, 0); + ASSERT_GT(setting_default, 0); + ASSERT_EQ(setting_off, 0); + ASSERT_GT(explicit_on, 0); + PASS(); +} + /* Same gap, opposite outcome: a project name that was never indexed has no * stored root_path to resolve, so it must still fail with the same clear * "repo_path is required" error rather than a resolver crash or silent @@ -24535,6 +24686,7 @@ SUITE(mcp) { RUN_TEST(tool_manage_adr_set_sections_is_advertised); RUN_TEST(tool_index_repository_reports_store_backed_adr); RUN_TEST(tool_index_repository_resolves_root_path_from_project_name_issue1211); + RUN_TEST(tool_index_repository_doc_candidates_switch); RUN_TEST(tool_index_repository_unknown_project_name_still_requires_repo_path); RUN_TEST(tool_index_repository_dot_uses_absolute_project_key_and_preserves_adr); RUN_TEST(index_repository_relative_path_uses_explicit_session_root); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index cb71d9ac53..19bb07e78a 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -19084,14 +19084,10 @@ static int doc_candidate_rows_for(cbm_store_t *s, const char *heading, char *top return ok ? n : -1; } -/* Doc -> code candidates end to end: a full index publishes a section's - * calibrated candidates (the best match first, a test function never, a - * changelog section none), and a delta-route reindex keeps them -- its proxy - * buffer cannot recompute them, so the clone's rows must survive. */ -TEST(pipeline_doc_candidates_published_and_kept_by_delta) { - char tmp[256]; - snprintf(tmp, sizeof(tmp), "/tmp/cbm_doccand_XXXXXX"); - ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); +/* The doc -> code candidate fixture: six Go config families, a test whose + * comment is the section's own text, and a README with a Mail-configuration + * section and a Changelog section. */ +static void write_doc_candidate_fixture(const char *tmp) { static const char *const subjects[] = {"User", "Server", "Client", "Cache", "Queue", "Mail"}; for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) { char file[64]; @@ -19118,6 +19114,17 @@ TEST(pipeline_doc_candidates_published_and_kept_by_delta) { "The mail config loader now validates missing fields and writes the mail " "config back to disk after parsing, so a parsed mail config file always " "carries every field it needs.\n"); +} + +/* Doc -> code candidates end to end: a full index publishes a section's + * calibrated candidates (the best match first, a test function never, a + * changelog section none), and a delta-route reindex keeps them -- its proxy + * buffer cannot recompute them, so the clone's rows must survive. */ +TEST(pipeline_doc_candidates_published_and_kept_by_delta) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_doccand_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + write_doc_candidate_fixture(tmp); char db[512]; snprintf(db, sizeof(db), "%s/doccand.db", tmp); @@ -19165,6 +19172,135 @@ TEST(pipeline_doc_candidates_published_and_kept_by_delta) { PASS(); } +/* Every doc_link_candidates row in a store (0 without the table); -1 on an + * error. */ +static int doc_candidate_row_count(cbm_store_t *s) { + sqlite3 *db = cbm_store_get_db(s); + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(db, + "SELECT count(*) FROM sqlite_master WHERE type='table' AND " + "name='doc_link_candidates'", + -1, &st, NULL) != SQLITE_OK) { + return -1; + } + bool has_table = sqlite3_step(st) == SQLITE_ROW && sqlite3_column_int(st, 0) > 0; + sqlite3_finalize(st); + if (!has_table) { + return 0; + } + if (sqlite3_prepare_v2(db, "SELECT count(*) FROM doc_link_candidates", -1, &st, NULL) != + SQLITE_OK) { + return -1; + } + int n = sqlite3_step(st) == SQLITE_ROW ? sqlite3_column_int(st, 0) : -1; + sqlite3_finalize(st); + return n; +} + +/* doc_candidates off: the same repository indexed with the switch off has no + * candidate rows and exactly the nodes and edges of the default run, which has + * candidates (the default is on). */ +TEST(pipeline_doc_candidates_off_stores_none_and_keeps_graph) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_doccand_off_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + write_doc_candidate_fixture(tmp); + /* The databases live outside the indexed tree, so both runs discover the + * same files. */ + char dbdir[256]; + snprintf(dbdir, sizeof(dbdir), "/tmp/cbm_doccand_db_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(dbdir)); + char db_on[512]; + char db_off[512]; + snprintf(db_on, sizeof(db_on), "%s/on.db", dbdir); + snprintf(db_off, sizeof(db_off), "%s/off.db", dbdir); + + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_on, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT(cbm_pipeline_doc_candidates_enabled(p)); + ASSERT_EQ(cbm_pipeline_run(p), 0); + char project[512]; + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + + p = cbm_pipeline_new(tmp, db_off, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + cbm_pipeline_set_doc_candidates_enabled(p, false); + ASSERT(!cbm_pipeline_doc_candidates_enabled(p)); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + cbm_store_t *on = cbm_store_open_path(db_on); + cbm_store_t *off = cbm_store_open_path(db_off); + ASSERT_NOT_NULL(on); + ASSERT_NOT_NULL(off); + int rows_on = doc_candidate_row_count(on); + int rows_off = doc_candidate_row_count(off); + int nodes_on = cbm_store_count_nodes(on, project); + int nodes_off = cbm_store_count_nodes(off, project); + int edges_on = cbm_store_count_edges(on, project); + int edges_off = cbm_store_count_edges(off, project); + cbm_store_close(on); + cbm_store_close(off); + ASSERT_GT(rows_on, 0); + ASSERT_EQ(rows_off, 0); + ASSERT_GT(nodes_on, 0); + ASSERT_EQ(nodes_off, nodes_on); + ASSERT_EQ(edges_off, edges_on); + th_rmtree(tmp); + th_rmtree(dbdir); + cbm_pipeline_incremental_test_reset_faults(); + PASS(); +} + +/* doc_candidates off on a delta-route reindex: the route that otherwise keeps + * the clone's candidate rows replaces them by none, so switching off leaves no + * stale candidates behind. */ +TEST(pipeline_doc_candidates_off_clears_rows_on_delta) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_doccand_delta_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + write_doc_candidate_fixture(tmp); + char db[512]; + snprintf(db, sizeof(db), "%s/doccand.db", tmp); + + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + cbm_store_t *s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + int before = doc_candidate_row_count(s); + cbm_store_close(s); + ASSERT_GT(before, 0); + + /* The same name-preserving edit as the delta test above: the closure + * repair route. */ + char edited[512]; + snprintf(edited, sizeof(edited), "%s/User_config.go", tmp); + FILE *ef = cbm_fopen(edited, "a"); + ASSERT_NOT_NULL(ef); + (void)fputs("\n// edited after the first index\n", ef); + (void)fclose(ef); + cbm_pipeline_incremental_test_reset_faults(); + p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + cbm_pipeline_set_doc_candidates_enabled(p, false); + ASSERT_EQ(cbm_pipeline_run(p), 0); + ASSERT_EQ(cbm_pipeline_incremental_test_last_route(), CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); + cbm_pipeline_free(p); + s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + int after = doc_candidate_row_count(s); + cbm_store_close(s); + ASSERT_EQ(after, 0); + th_rmtree(tmp); + cbm_pipeline_incremental_test_reset_faults(); + PASS(); +} + /* One Python family per subject: parse / load / validate / write, each with * a docstring, so the functions' corpus has the subject words to match. */ static void write_py_family(const char *base, const char *subject) { @@ -20507,6 +20643,8 @@ SUITE(pipeline) { RUN_TEST(pipeline_semantic_pair_signals_never_change_the_graph); RUN_TEST(pipeline_semantic_edges_carry_p); RUN_TEST(pipeline_doc_candidates_published_and_kept_by_delta); + RUN_TEST(pipeline_doc_candidates_off_stores_none_and_keeps_graph); + RUN_TEST(pipeline_doc_candidates_off_clears_rows_on_delta); RUN_TEST(pipeline_doc_candidates_skip_exact_links_and_use_format_curves); RUN_TEST(pipeline_doc_candidates_home_position_and_kinds); RUN_TEST(pipeline_doc_candidates_diagram_family_curves);