Merge pull request DeusData#1723 from liuchong/feat/index-resource-discovery

feat(index): add opt-in discovery resource limits

Author: DeusData

Makefile.cbm

@@ -325,6 +325,7 @@ FOUNDATION_SRCS = \
     src/foundation/profile.c \
     src/foundation/dump_verify.c \
     src/foundation/limits.c \
+    src/foundation/index_policy.c \
     src/foundation/subprocess.c \
     src/foundation/sha256.c \
     src/foundation/secure_random.c \
@@ -617,6 +618,7 @@ TEST_FOUNDATION_SRCS = \
     tests/test_str_intern.c \
     tests/test_log.c \
     tests/test_str_util.c \
+    tests/test_index_policy.c \
     tests/test_workspace.c \
     tests/test_platform.c \
     tests/test_diagnostics.c \

README.md

@@ -733,9 +733,15 @@ codebase-memory-mcp config set auto_index true           # auto-index on session
 codebase-memory-mcp config set auto_index_limit 50000    # max files for auto-index
 codebase-memory-mcp config set auto_watch false          # don't register background git watcher (default: true)
 codebase-memory-mcp config set watcher_enabled false     # stop the watcher thread entirely (default: true)
+codebase-memory-mcp config set index_max_files 250000    # optional per-index source-file limit
+codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit
 codebase-memory-mcp config reset auto_index              # reset to default
 ```
 
+The two `index_max_*` settings default to `off`. Exceeding one fails the complete
+index attempt rather than publishing a partial graph; an existing serving index
+is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md).
+
 ### Environment Variables
 
 | Variable | Default | Description |

docs/CONFIGURATION.md

@@ -89,6 +89,8 @@ Current keys:
 | `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. |
 | `auto_watch` | `true` | Register the session's project with the background git watcher on connect. Set `false` to keep a session from registering its project (the watcher still runs for other projects). |
 | `watcher_enabled` | `true` | Master switch for the background watcher subsystem. Set `false` to stop the watcher from starting at all — no poll thread and no project registration. Reindex manually with `index_repository` when disabled. |
+| `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. |
+| `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. |
 
 > **`watcher_enabled` vs `auto_watch`.** `watcher_enabled` controls whether the
 > watcher *subsystem* starts at all (the background poll thread). `auto_watch` is
@@ -116,6 +118,14 @@ Current keys:
 > `auto_index` still runs, and `index_repository` stays available for manual
 > reindexing.
 
+The two `index_max_*` settings are independent and disabled by default. They
+apply to explicit indexing, automatic indexing, and watcher re-indexing, but not
+to `cross-repo-intelligence`, which does not scan repository source files.
+Equality is allowed; exceeding either setting fails the complete index request
+and preserves any previously serving database. See
+[Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and
+error-response details.
+
 ## 3. UI Settings
 
 The optional built-in graph UI stores its settings in:

docs/INDEX_RESOURCE_LIMITS.md

@@ -0,0 +1,68 @@
+# Index resource limits
+
+Index resource limits are optional operator controls for repositories whose
+discovery breadth is not known in advance. They are disabled by default so
+existing large-repository workloads retain their current behavior.
+
+## Discovery settings
+
+| Key | Default | Accepted value | Protects |
+|---|---:|---:|---|
+| `index_max_files` | `off` | `off` or `1..10000000` | Accepted source-file count |
+| `index_max_source_mb` | `off` | `off` or `1..1048576` | Accepted source-file bytes |
+
+Set or reset them with the normal configuration command:
+
+```bash
+codebase-memory-mcp config set index_max_files 250000
+codebase-memory-mcp config set index_max_source_mb 16384
+codebase-memory-mcp config reset index_max_files
+```
+
+Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero,
+negative values, suffixes, trailing characters, and values outside the stated
+ranges are rejected without changing the stored value.
+
+## Counting and failure semantics
+
+`index_max_files` counts a file only after it passes directory pruning, ignore
+rules, filename and suffix filters, language detection, and the existing
+per-file size rule. `index_max_source_mb` sums the filesystem sizes of that same
+accepted set.
+
+Equality is allowed. The first file or byte that makes an observed value greater
+than its limit stops discovery. CBM discards the partial file list and does not
+publish a partial graph as a complete index.
+
+For an explicit MCP request the error payload contains:
+
+```json
+{
+  "status": "error",
+  "code": "resource_limit_exceeded",
+  "stage": "discovery",
+  "resource": "files",
+  "observed": 250001,
+  "limit": 250000,
+  "unit": "files",
+  "retryable": true,
+  "serving_index_preserved": true,
+  "message": "Index discovery exceeded index_max_files"
+}
+```
+
+The previous database remains available because publication occurs only after a
+complete discovery and successful staged build. If no previous database exists,
+`serving_index_preserved` is false.
+
+## Trust and compatibility
+
+Limits are read from the CLI-managed `_config.db`; they are not MCP request
+arguments. A supervised parent replaces any caller-supplied internal policy
+before spawning its worker, and the worker rejects a missing or incomplete
+parent policy.
+
+These settings do not replace or increase `auto_index_limit`, change the 512 MiB
+single-file cap, alter workspace-root authorization, or affect
+`cross-repo-intelligence`. With both settings `off`, discovery follows the
+existing unbounded path.

scripts/test-runtime.sh

@@ -105,6 +105,16 @@ _cbm_test_runtime_daemon() {
         CBM_CACHE_DIR="$_CBM_TEST_RUNTIME_PRODUCT_CACHE" "$1" daemon "$2"
 }
 
+# The supervisor resolves one resource policy per index and hands it to the
+# worker in argv; a worker that finds no complete policy refuses to start rather
+# than index unbounded. A shell test that spawns `cli --index-worker` itself
+# stands in for the supervisor and owes the worker the same object. Mirrors
+# cbm_mcp_index_policy_add_to_args: every key in cbm_index_policy_key_at, and
+# nothing else.
+cbm_test_index_worker_policy_json() {
+    printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"}'
+}
+
 cbm_test_runtime_cleanup() {
     local binary="${1:-}" root="$_CBM_TEST_RUNTIME_CREATED_ROOT"
     local name="${_CBM_TEST_RUNTIME_CREATED_ROOT##*/}" runtime_entry="" active=0

src/cli/cli.c

@@ -7404,6 +7404,25 @@ bool cbm_config_watcher_enabled(cbm_config_t *cfg) {
     return cbm_config_get_bool(cfg, CBM_CONFIG_WATCHER_ENABLED, true);
 }
 
+bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy,
+                                  char *error, size_t error_size) {
+    if (!cfg || !policy) {
+        if (error && error_size > 0) {
+            (void)snprintf(error, error_size, "index resource configuration is unavailable");
+        }
+        return false;
+    }
+    cbm_index_policy_init(policy);
+    for (size_t index = 0; index < cbm_index_policy_key_count(); index++) {
+        const char *key = cbm_index_policy_key_at(index);
+        const char *value = cbm_config_get(cfg, key, cbm_index_policy_default_value(key));
+        if (!cbm_index_policy_set(policy, key, value, error, error_size)) {
+            return false;
+        }
+    }
+    return true;
+}
+
 /* ── Config CLI subcommand ────────────────────────────────────── */
 
 /* THE config-key table. list, get, help, and key validation all read this one
@@ -7428,6 +7447,8 @@ static const config_key_def_t CONFIG_KEYS[] = {
     {CBM_CONFIG_UI_LANG, "auto", "Pin graph UI language: en, zh, or auto"},
     {CBM_CONFIG_UI_ENABLED, "false", "Serve the graph UI on a loopback HTTP port"},
     {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"},
+    {CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"},
+    {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"},
 };
 
 /* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing
@@ -7453,6 +7474,31 @@ static bool config_key_is_ui(const char *key) {
     return key && (strcmp(key, CBM_CONFIG_UI_ENABLED) == 0 || strcmp(key, CBM_CONFIG_UI_PORT) == 0);
 }
 
+static bool config_key_is_index_policy(const char *key) {
+    return key && (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0 ||
+                   strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0);
+}
+
+static int config_index_policy_write(cbm_config_t *config, const char *key, const char *value) {
+    cbm_index_resource_policy_t candidate;
+    cbm_index_policy_init(&candidate);
+    char error[CLI_BUF_256];
+    if (!cbm_index_policy_set(&candidate, key, value, error, sizeof(error))) {
+        (void)fprintf(stderr, "error: %s\n", error);
+        return CLI_ERR;
+    }
+    int rc = cbm_config_set(config, key, value);
+    if (rc != 0) {
+        /* The caller suppresses its own message for policy keys because this
+         * helper names the precise reason. That is only true if the helper
+         * speaks on every failure it can return: a validated value whose write
+         * then fails -- a locked or read-only _config.db -- used to exit
+         * non-zero having printed nothing at all. */
+        (void)fprintf(stderr, "error: failed to set %s\n", key);
+    }
+    return rc;
+}
+
 static void config_ui_read(const char *key, char *out, size_t out_sz) {
     cbm_ui_config_t ui;
     cbm_ui_config_load(&ui);
@@ -7590,10 +7636,16 @@ int cbm_cmd_config(int argc, char **argv) {
                 rc = CLI_TRUE;
             }
         } else {
-            if (cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]) == 0) {
+            int set_rc =
+                config_key_is_index_policy(argv[CLI_SKIP_ONE])
+                    ? config_index_policy_write(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN])
+                    : cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]);
+            if (set_rc == 0) {
                 printf("%s = %s\n", argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]);
             } else {
-                (void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]);
+                if (!config_key_is_index_policy(argv[CLI_SKIP_ONE])) {
+                    (void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]);
+                }
                 rc = CLI_TRUE;
             }
         }

src/cli/cli.h

@@ -13,6 +13,8 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include "foundation/index_policy.h"
+
 typedef struct cbm_mcp_server cbm_mcp_server_t;
 
 /* ── Version ──────────────────────────────────────────────────── */
@@ -420,6 +422,11 @@ int cbm_config_set(cbm_config_t *cfg, const char *key, const char *value);
 /* Delete a config key. Returns 0 on success. */
 int cbm_config_delete(cbm_config_t *cfg, const char *key);
 
+/* Load and validate the operator-controlled discovery policy. Invalid stored
+ * values fail closed instead of silently disabling a guard. */
+bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy,
+                                  char *error, size_t error_size);
+
 /* Well-known config keys */
 #define CBM_CONFIG_AUTO_INDEX "auto_index"
 #define CBM_CONFIG_AUTO_INDEX_LIMIT "auto_index_limit"

src/daemon/application.c

@@ -208,7 +208,8 @@ static bool application_unique_recovery_file(char out[APPLICATION_PATH_CAP], con
 static bool application_update_reap(cbm_daemon_application_t *application, bool wait,
                                     uint32_t timeout_ms);
 static void *application_job_thread(void *opaque);
-static char *application_auto_index_args(const char *root_path);
+static char *application_auto_index_args(cbm_daemon_application_t *application,
+                                         const char *root_path);
 static cbm_daemon_application_job_t *application_job_subscribe_locked(
     cbm_daemon_application_t *application, const char *project_key, const char *root_path,
     const char *args_json, application_job_subscribe_status_t *status_out);
@@ -1455,7 +1456,7 @@ static void application_auto_index_retry_pending_locked(cbm_daemon_application_t
             application_refresh_watch_locked(session);
             continue;
         }
-        char *args = application_auto_index_args(root_path);
+        char *args = application_auto_index_args(application, root_path);
         if (!args) {
             continue;
         }
@@ -2005,15 +2006,36 @@ static bool application_update_reap(cbm_daemon_application_t *application, bool
     }
 }
 
-static char *application_auto_index_args(const char *root_path) {
+static bool application_index_args_add_policy(cbm_daemon_application_t *application,
+                                              yyjson_mut_doc *document, yyjson_mut_val *root) {
+    cbm_config_t *owned_config = NULL;
+    cbm_config_t *config = application ? application->config : NULL;
+    if (!config) {
+        owned_config = cbm_config_open(cbm_resolve_cache_dir());
+        config = owned_config;
+    }
+    cbm_index_resource_policy_t policy;
+    char error[CBM_SZ_256] = {0};
+    bool loaded = cbm_config_load_index_policy(config, &policy, error, sizeof(error));
+    cbm_config_close(owned_config);
+    if (!loaded) {
+        cbm_log_error("daemon.index.policy", "error", error);
+        return false;
+    }
+    return cbm_mcp_index_policy_add_to_args(document, root, &policy);
+}
+
+static char *application_auto_index_args(cbm_daemon_application_t *application,
+                                         const char *root_path) {
     yyjson_mut_doc *document = yyjson_mut_doc_new(NULL);
     yyjson_mut_val *root = document ? yyjson_mut_obj(document) : NULL;
     if (!document || !root) {
         yyjson_mut_doc_free(document);
         return NULL;
     }
     yyjson_mut_doc_set_root(document, root);
-    char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path)
+    char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) &&
+                         application_index_args_add_policy(application, document, root)
                      ? yyjson_mut_write(document, 0, NULL)
                      : NULL;
     yyjson_mut_doc_free(document);
@@ -2060,7 +2082,7 @@ static void application_background_initialize_impl(cbm_daemon_application_sessio
                      files);
     }
     bool args_required = auto_index_candidate && within_auto_index_limit;
-    char *args = args_required ? application_auto_index_args(root_path) : NULL;
+    char *args = args_required ? application_auto_index_args(application, root_path) : NULL;
     application_jobs_reap_completed(application);
     cbm_mutex_lock(&application->mutex);
     if (application->stopping || application_request_cancelled_locked(session)) {
@@ -3458,7 +3480,8 @@ static int application_background_index(cbm_daemon_application_t *application,
         return -1;
     }
     yyjson_mut_doc_set_root(document, root);
-    bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root);
+    bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root) &&
+                   application_index_args_add_policy(application, document, root);
     char *default_project = cbm_project_name_from_path(canonical_root);
     bool custom_project =
         project_name[0] && (!default_project || strcmp(default_project, project_name) != 0);