Break-glass merge PR #2610

Author: DeusData

README.md

@@ -793,6 +793,7 @@ codebase-memory-mcp config list                          # show all settings
 codebase-memory-mcp config set auto_index true           # auto-index on session start
 codebase-memory-mcp config set auto_index_limit 50000    # max files for auto-index
 codebase-memory-mcp config set auto_watch false          # don't register background git watcher (default: true)
+codebase-memory-mcp config set doc_candidates false      # skip doc<->code candidate scoring on huge repos (default: true)
 codebase-memory-mcp config set watcher_enabled false     # stop the watcher thread entirely (default: true)
 codebase-memory-mcp config set watch_non_git true        # also poll non-git project roots (default: false)
 codebase-memory-mcp config set index_max_files 250000    # optional per-index source-file limit

docs/CONFIGURATION.md

@@ -88,6 +88,7 @@ Current keys:
 | `auto_index` | `false` | Automatically index new projects when an MCP session starts. |
 | `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. |
 | `auto_watch` | `true` | Register the session's project with the background git watcher on connect. Set `false` to keep a session from registering its project (the watcher still runs for other projects). |
+| `doc_candidates` | `true` | Score documentation sections against code to store doc↔code candidates (the possible targets of a section, and the sections that may describe a function or file, each with a judged probability). The scoring grows with sections × functions: under a second up to ~100k functions, ~15 s at ~400k, ~3 minutes on the Linux kernel. Set `false` to skip it; no candidates are stored and the rest of the graph is unchanged. `index_repository` also takes `doc_candidates` for a single index, which wins over this setting. Applies from the next index. |
 | `watcher_enabled` | `true` | Master switch for the background watcher subsystem. Set `false` to stop the watcher from starting at all — no poll thread and no project registration. Reindex manually with `index_repository` when disabled. Only an open MCP session's own project is ever watched; `index_status` reports the current state in its `watch` object. |
 | `watch_non_git` | `false` | Also poll project roots that are **not git repositories**. By default the watcher only follows git projects, so a project indexed from a plain directory is never refreshed after its first index — reindex it manually with `index_repository`. Set `true` to poll such roots on the same adaptive cadence with a file-tree scan: the indexer's own discovery walk (same skip lists, `.gitignore` and `.cbmignore` rules) hashed over each file's path, size and mtime. Any change reindexes once; paths the indexer skips (including cbm's own `.codebase-memory/` output) never trigger. The first poll after the daemon starts reindexes each such project once, since nothing records which tree state the index holds. The scan walks the whole tree every poll, so it costs more than git polling on very large trees. Read once when the daemon starts, like `watcher_enabled`. |
 | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. |

src/cli/cli.h

@@ -452,6 +452,7 @@ bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t
 #define CBM_CONFIG_AUTO_INDEX "auto_index"
 #define CBM_CONFIG_AUTO_INDEX_LIMIT "auto_index_limit"
 #define CBM_CONFIG_AUTO_WATCH "auto_watch"
+#define CBM_CONFIG_DOC_CANDIDATES "doc_candidates"
 #define CBM_CONFIG_UI_LANG "ui-lang"
 #define CBM_CONFIG_WATCHER_ENABLED "watcher_enabled"
 #define CBM_CONFIG_WATCH_NON_GIT "watch_non_git"

src/cli/runtime_settings.c

@@ -46,6 +46,10 @@ static const setting_t settings[] = {
     CFG("auto_watch", "Automatic watching",
         "Register a background Git watcher when an MCP session connects.", "boolean", "true", 0, 0,
         NULL, "src/mcp/mcp.c", "next-session"),
+    CFG("doc_candidates", "Doc-code candidates",
+        "Score documentation sections against code for doc-code candidates. Costly on very large "
+        "repositories; off stores none.",
+        "boolean", "true", 0, 0, NULL, "src/mcp/mcp.c", "next-session"),
     CFG("watcher_enabled", "Background watcher", "Enable the daemon's background watcher thread.",
         "boolean", "true", 0, 0, NULL, "src/daemon/application.c", "restart"),
     {"ui-lang", "Interface language", "Language preference used by subsequent HTTP requests.",

src/mcp/mcp.c

@@ -548,6 +548,8 @@ static const tool_def_t TOOLS[] = {
      "\"Name override; Non-ASCII bytes are encoded; unsafe characters normalized.\"},"
      "\"persistence\":{\"type\":\"boolean\",\"default\":false,\"description\":"
      "\"Write .codebase-memory/graph.db.zst.\"},"
+     "\"doc_candidates\":{\"type\":\"boolean\",\"description\":"
+     "\"Doc-code candidates; default: the doc_candidates setting (on).\"},"
      "\"async\":{\"type\":\"boolean\",\"default\":false,\"description\":"
      "\"Start the index in the background and return immediately; it keeps running if this "
      "call is cancelled or times out. Not with status or cross-repo-intelligence. Refused for a "
@@ -11738,14 +11740,32 @@ static bool resolve_session_repo_path(cbm_mcp_server_t *srv, char **repo_path) {
     return true;
 }
 
+/* doc_candidates for one index: an explicit boolean argument wins, else the
+ * doc_candidates setting (default on). The parent resolves it once and passes
+ * the decided value on, so a daemon job or supervised worker gets it as an
+ * argument instead of reading a config of its own. */
+static bool index_doc_candidates_enabled(cbm_mcp_server_t *srv, const char *args) {
+    yyjson_doc *doc = args ? yyjson_read(args, strlen(args), 0) : NULL;
+    yyjson_val *val = doc ? yyjson_obj_get(yyjson_doc_get_root(doc), "doc_candidates") : NULL;
+    bool explicit_value = val && yyjson_is_bool(val);
+    bool enabled = explicit_value ? yyjson_get_bool(val) : true;
+    yyjson_doc_free(doc);
+    if (!explicit_value && srv && srv->config) {
+        enabled = cbm_config_get_bool(srv->config, CBM_CONFIG_DOC_CANDIDATES, true);
+    }
+    return enabled;
+}
+
 /* Preserve every index option while replacing all caller-supplied repo_path
  * keys with the one canonical path that was actually authorized. A non-empty
  * project_name likewise replaces any caller "name" (#2134: the project key the
  * parent resolved for this root travels to the daemon job / worker verbatim,
- * so lease key and written index cannot diverge). */
+ * so lease key and written index cannot diverge). doc_candidates is replaced
+ * by the value the parent resolved. */
 static char *index_args_with_repo_path(const char *args, const char *canonical_repo_path,
                                        const char *project_name,
-                                       const cbm_index_resource_policy_t *policy) {
+                                       const cbm_index_resource_policy_t *policy,
+                                       bool doc_candidates) {
     if (!args || !canonical_repo_path || !policy) {
         return NULL;
     }
@@ -11783,9 +11803,13 @@ static char *index_args_with_repo_path(const char *args, const char *canonical_r
     while (has_name && yyjson_mut_obj_get(copy_root, "name")) {
         (void)yyjson_mut_obj_remove_key(copy_root, "name");
     }
+    while (yyjson_mut_obj_get(copy_root, "doc_candidates")) {
+        (void)yyjson_mut_obj_remove_key(copy_root, "doc_candidates");
+    }
     if ((has_name && !yyjson_mut_obj_add_strcpy(copy, copy_root, "name", project_name)) ||
         !yyjson_mut_obj_add_strcpy(copy, copy_root, "repo_path", canonical_repo_path) ||
-        !cbm_mcp_index_policy_add_to_args(copy, copy_root, policy)) {
+        !cbm_mcp_index_policy_add_to_args(copy, copy_root, policy) ||
+        !yyjson_mut_obj_add_bool(copy, copy_root, "doc_candidates", doc_candidates)) {
         yyjson_mut_doc_free(copy);
         return NULL;
     }
@@ -12151,12 +12175,13 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
         index_args_free(repo_path, mode_str, name_override);
         return cbm_mcp_text_result(policy_error, true);
     }
+    bool doc_candidates = index_doc_candidates_enabled(srv, args);
 
     /* A daemon session delegates the one physical write to its shared job
      * registry only after path canonicalization and workspace authorization. */
     if (srv->index_executor) {
-        char *worker_args =
-            index_args_with_repo_path(args, repo_path, name_override, &resource_policy);
+        char *worker_args = index_args_with_repo_path(args, repo_path, name_override,
+                                                      &resource_policy, doc_candidates);
         char *coordinated = worker_args ? srv->index_executor(srv->index_executor_context,
                                                               repo_path, worker_args, async_mode)
                                         : NULL;
@@ -12184,8 +12209,8 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
      * installs the same guard before running the in-process pipeline. A marked
      * host fails closed if preparation or worker startup cannot complete. */
     if (cbm_index_supervisor_should_wrap()) {
-        char *worker_args =
-            index_args_with_repo_path(args, repo_path, name_override, &resource_policy);
+        char *worker_args = index_args_with_repo_path(args, repo_path, name_override,
+                                                      &resource_policy, doc_candidates);
         if (!worker_args) {
             free(mutation_project);
             index_args_free(repo_path, mode_str, name_override);
@@ -12249,6 +12274,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
     free(name_override);
     cbm_pipeline_set_persistence(p, persistence);
     cbm_pipeline_set_resource_policy(p, &resource_policy);
+    cbm_pipeline_set_doc_candidates_enabled(p, doc_candidates);
 
     char *project_name = heap_strdup(cbm_pipeline_project_name(p));
 
@@ -19957,6 +19983,7 @@ static void *autoindex_thread(void *arg) {
         cbm_log_warn("autoindex.err", "msg", "pipeline_create_failed");
         return NULL;
     }
+    cbm_pipeline_set_doc_candidates_enabled(p, index_doc_candidates_enabled(srv, NULL));
 
     /* Block until any concurrent pipeline finishes */
     cbm_pipeline_lock();

src/pipeline/pass_semantic_edges.c

@@ -3029,6 +3029,12 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *
     if (!corpus || func_count <= 0) {
         return;
     }
+    /* doc_candidates off: no scoring and no rows (the scoring grows with
+     * sections x functions; on the Linux kernel it is ~174 s of the index). */
+    if (!cbm_pipeline_doc_candidates_enabled(ctx->pipeline)) {
+        cbm_log_info("pass.semantic.doc_candidates", "skipped", "doc_candidates_off");
+        return;
+    }
     const cbm_gbuf_node_t **found = NULL;
     int nsec = 0;
     if (cbm_gbuf_find_by_label(gbuf, "Section", &found, &nsec) != 0 || nsec == 0) {

src/pipeline/pipeline.c

@@ -278,6 +278,8 @@ struct cbm_pipeline {
      * cbm_pipeline_free). NULL when the pass found none or did not run. */
     cbm_doc_candidate_t *doc_candidates;
     int doc_candidate_count;
+    /* doc_candidates setting off (calloc'd false = on, the default). */
+    bool doc_candidates_off;
 
     /* Deterministic test-only seam at the final publication boundary. Kept
      * per pipeline so concurrent test/process activity cannot cross-trigger. */
@@ -874,6 +876,16 @@ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled) {
     }
 }
 
+void cbm_pipeline_set_doc_candidates_enabled(cbm_pipeline_t *p, bool enabled) {
+    if (p) {
+        p->doc_candidates_off = !enabled;
+    }
+}
+
+bool cbm_pipeline_doc_candidates_enabled(const cbm_pipeline_t *p) {
+    return !p || !p->doc_candidates_off;
+}
+
 void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p,
                                       const cbm_index_resource_policy_t *policy) {
     if (p && policy) {

src/pipeline/pipeline.h

@@ -56,6 +56,13 @@ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled);
 /* Apply a validated discovery resource policy. The value is copied. */
 void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, const cbm_index_resource_policy_t *policy);
 
+/* Doc -> code candidates (the doc_candidates setting; default on). Off skips
+ * scoring documentation sections against code and leaves the project with no
+ * doc_link_candidates rows; nodes and edges are unchanged. The scoring grows
+ * with sections x functions, which is the cost on very large repositories. */
+void cbm_pipeline_set_doc_candidates_enabled(cbm_pipeline_t *p, bool enabled);
+bool cbm_pipeline_doc_candidates_enabled(const cbm_pipeline_t *p);
+
 /* Copy the exact discovery violation from the most recent run. */
 void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p,
                                          cbm_index_resource_violation_t *violation);

src/pipeline/pipeline_incremental.c

@@ -3172,7 +3172,9 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
             .doc_link_rows = doc_rows,
             .doc_link_row_count = doc_row_count,
             .doc_links_failed = doc_failed,
-            .doc_candidates_in_place = true, /* the clone keeps the previous rows */
+            /* The clone keeps the previous rows; with doc_candidates off they
+             * are replaced by none, so turning it off leaves no stale rows. */
+            .doc_candidates_in_place = cbm_pipeline_doc_candidates_enabled(p),
         };
         cbm_store_close(staging);
         staging = NULL;