Skip to content

Commit 7884ccb

Browse files
authored
Merge pull request DeusData#853 from DeusData/feat/index-mem-subprocess-isolation
fix(index): supervised-subprocess RSS isolation for background auto-index + watcher
2 parents a960d3d + 0814af1 commit 7884ccb

7 files changed

Lines changed: 637 additions & 3 deletions

File tree

‎src/foundation/mem.c‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,15 @@ void cbm_mem_init(double ram_fraction) {
137137
mi_option_set(mi_option_arena_eager_commit, 0);
138138
mi_option_set(mi_option_purge_decommits, SKIP_ONE);
139139
mi_option_set(mi_option_purge_delay, 0); /* immediate purge, no 1s delay */
140+
/* v3 (#832): reclaim abandoned pages on ANY thread's free (=1), restoring the
141+
* v2 behaviour. mimalloc v3 defaults page_reclaim_on_free=0, so pages a worker
142+
* thread abandons at exit are NOT reclaimed when the main thread later frees
143+
* their blocks (and mi_collect cannot touch abandoned pages) — RSS then
144+
* ratchets across repeated in-process index cycles. The supervised subprocess
145+
* is the primary cure (the child returns 100% RSS on exit); this is the
146+
* in-process fallback for any path that stays in-process (kill switch,
147+
* spawn-fail degrade, embedders). */
148+
mi_option_set(mi_option_page_reclaim_on_free, 1);
140149

141150
/* CBM_MEM_BUDGET_MB env override (memory analogue of CBM_WORKERS).
142151
* Lets users cap the budget directly without an enclosing cgroup —

‎src/main.c‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,23 @@ static int watcher_index_fn(const char *project_name, const char *root_path, voi
169169

170170
cbm_log_info("watcher.reindex", "project", project_name, "path", root_path);
171171

172+
/* #832: route the re-index through the supervised worker subprocess so this
173+
* long-lived server process hands its RSS back to the OS on every cycle
174+
* instead of ratcheting (mimalloc v3 does not reclaim pages that worker
175+
* threads abandon at exit). The child writes the DB; the parent only needs the
176+
* return code. The pipeline lock (already held) still serialises re-indexes.
177+
* Degrade to the in-process pipeline when the supervisor is off (kill switch)
178+
* or the spawn fails. */
179+
if (cbm_index_supervisor_should_wrap()) {
180+
char *resp = cbm_mcp_index_run_supervised_path(root_path);
181+
if (resp) {
182+
free(resp);
183+
cbm_pipeline_unlock();
184+
return 0;
185+
}
186+
/* resp == NULL → spawn-failure degrade → fall through to in-process. */
187+
}
188+
172189
cbm_pipeline_t *p = cbm_pipeline_new(root_path, NULL, CBM_MODE_FULL);
173190
if (!p) {
174191
cbm_pipeline_unlock();

‎src/mcp/mcp.c‎

Lines changed: 56 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3481,8 +3481,14 @@ static char *build_worker_failure_response(const char *args, cbm_proc_outcome_t
34813481
}
34823482

34833483
/* Drop the cached store so the next query reopens whatever the worker wrote (each
3484-
* worker is a fresh process that deletes + recreates the .db). */
3484+
* worker is a fresh process that deletes + recreates the .db). NULL-safe: the
3485+
* background watcher path (main.c) has no MCP server / cached store — the child
3486+
* writes the DB and the parent only needs the return code, so there is nothing
3487+
* to invalidate. */
34853488
static void supervisor_invalidate_store(cbm_mcp_server_t *srv) {
3489+
if (!srv) {
3490+
return;
3491+
}
34863492
if (srv->owns_store && srv->store) {
34873493
cbm_store_close(srv->store);
34883494
srv->store = NULL;
@@ -3687,6 +3693,34 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) {
36873693
return build_worker_failure_response(args, last_outcome);
36883694
}
36893695

3696+
/* Build a minimal {"repo_path": "<root>"} args object (path safely escaped) and
3697+
* run it through index_run_supervised. Shared by the session auto-index (srv
3698+
* present → its cached store is invalidated) and the watcher re-index (srv NULL).
3699+
* Returns the worker's response string (caller frees) or NULL to degrade. */
3700+
static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_path) {
3701+
if (!root_path || !root_path[0]) {
3702+
return NULL;
3703+
}
3704+
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
3705+
yyjson_mut_val *root = yyjson_mut_obj(doc);
3706+
yyjson_mut_doc_set_root(doc, root);
3707+
yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path);
3708+
char *args = yy_doc_to_str(doc);
3709+
yyjson_mut_doc_free(doc);
3710+
if (!args) {
3711+
return NULL;
3712+
}
3713+
char *resp = index_run_supervised(srv, args);
3714+
free(args);
3715+
return resp;
3716+
}
3717+
3718+
/* Public entry (see mcp.h): the watcher re-index in main.c has no MCP server, so
3719+
* it reaches the supervised runner through this srv-less wrapper. */
3720+
char *cbm_mcp_index_run_supervised_path(const char *root_path) {
3721+
return index_run_supervised_path(NULL, root_path);
3722+
}
3723+
36903724
static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
36913725
/* Supervisor gate: run the index in a crash/hang-isolating worker subprocess
36923726
* unless this process IS the worker or the kill switch (CBM_INDEX_SUPERVISOR=0)
@@ -5662,6 +5696,26 @@ static void *autoindex_thread(void *arg) {
56625696

56635697
cbm_log_info("autoindex.start", "project", srv->session_project, "path", srv->session_root);
56645698

5699+
/* #832: prefer the supervised worker subprocess. Indexing the whole session in
5700+
* this long-lived server thread ratchets RSS (mimalloc v3 does not reclaim the
5701+
* pages worker threads abandon at exit); running it in a child that exits hands
5702+
* 100% of that memory back to the OS every cycle. Degrade to the in-process
5703+
* pipeline below when the supervisor is off (kill switch) or the spawn fails. */
5704+
if (cbm_index_supervisor_should_wrap()) {
5705+
char *resp = index_run_supervised_path(srv, srv->session_root);
5706+
if (resp) {
5707+
free(resp);
5708+
cbm_log_info("autoindex.done", "project", srv->session_project, "mode", "supervised");
5709+
/* Register with watcher for ongoing change detection — gated on
5710+
* auto_watch (#849), same as the in-process branch below. A bare
5711+
* `if (srv->watcher)` would register even when the user set
5712+
* `config set auto_watch false`, since srv->watcher is always set. */
5713+
register_watcher_if_enabled(srv);
5714+
return NULL;
5715+
}
5716+
/* resp == NULL → spawn-failure degrade → fall through to in-process. */
5717+
}
5718+
56655719
cbm_pipeline_t *p = cbm_pipeline_new(srv->session_root, NULL, CBM_MODE_FULL);
56665720
if (!p) {
56675721
cbm_log_warn("autoindex.err", "msg", "pipeline_create_failed");
@@ -5674,7 +5728,7 @@ static void *autoindex_thread(void *arg) {
56745728
cbm_pipeline_unlock();
56755729

56765730
cbm_pipeline_free(p);
5677-
cbm_mem_collect(); /* return mimalloc pages to OS after indexing */
5731+
cbm_mem_collect(); /* return mimalloc pages to OS after indexing (in-process only) */
56785732

56795733
if (rc == 0) {
56805734
cbm_log_info("autoindex.done", "project", srv->session_project);

‎src/mcp/mcp.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,18 @@ char *cbm_mcp_server_handle(cbm_mcp_server_t *srv, const char *line);
116116
/* Handle a tools/call request. Returns MCP tool result JSON. */
117117
char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const char *args_json);
118118

119+
/* ── Supervised background index (RSS isolation, #832) ────────── */
120+
121+
/* Run a full index of root_path in a supervised worker SUBPROCESS (the same
122+
* crash/hang-isolating runner used by handle_index_repository), so the child
123+
* returns 100% of its RSS to the OS on exit instead of ratcheting the long-lived
124+
* parent. Builds {"repo_path": root_path} internally. Returns the worker's
125+
* response string (caller frees) on success, or NULL to signal the caller must
126+
* degrade to the in-process path (kill switch set, spawn failure, or the process
127+
* is not a supervisor host). This is the shared entry the watcher re-index
128+
* (main.c) and the session auto-index (mcp.c) route through. */
129+
char *cbm_mcp_index_run_supervised_path(const char *root_path);
130+
119131
/* ── Idle store eviction ──────────────────────────────────────── */
120132

121133
/* Evict the cached project store if idle for more than timeout_s seconds.

‎tests/repro/issue832_rss.py‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
#!/usr/bin/env python3
2+
"""
3+
issue832_rss.py -- direct RSS reproduction for #832 (NON-GATING, manual repro tier).
4+
5+
WHY THIS IS NOT A C UNIT TEST
6+
-----------------------------
7+
The RSS ratchet is a property of mimalloc v3's abandoned-page handling
8+
(page_reclaim_on_free=0): pages a worker THREAD abandons at exit are not
9+
reclaimed when the main thread later frees their blocks. That only manifests in
10+
the PROD binary, which links mimalloc as the global allocator (Makefile.cbm:
11+
MI_OVERRIDE=1). The C test-runner and the C repro-runner are built CRT+ASan
12+
(MI_OVERRIDE=0), so mimalloc is inert there and cbm_mem_rss() falls back to
13+
os_rss() -- a C test would be VACUOUS. Hence this drives the real
14+
`build/c/codebase-memory-mcp` server over stdio and samples its RSS from `ps`.
15+
16+
WHAT IT SHOWS
17+
-------------
18+
A long-lived MCP server is driven through K index_repository cycles of the same
19+
fixture. The in-process pipeline (CBM_INDEX_SUPERVISOR=0) is the pre-#832-fix
20+
background-path behaviour: RSS RATCHETS across cycles. The supervised subprocess
21+
path (default) is the fix: each child returns 100% of its RSS on exit, so the
22+
long-lived parent stays ~FLAT. The auto-index (mcp.c) and watcher re-index
23+
(main.c) paths now route through that same supervised subprocess, so they inherit
24+
this flat profile; the deterministic routing proof is the GATING guard
25+
tests/test_mcp.c::index_bg_paths_route_through_supervisor_issue832.
26+
27+
Inherently noisy (allocator/OS dependent) -> thresholds are generous and this is
28+
NOT wired into `make test` / `ci-ok`. Run manually:
29+
make -f Makefile.cbm cbm
30+
python3 tests/repro/issue832_rss.py
31+
"""
32+
import json
33+
import os
34+
import shutil
35+
import subprocess
36+
import sys
37+
import tempfile
38+
39+
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
40+
BINARY = os.path.join(ROOT, "build", "c", "codebase-memory-mcp")
41+
CYCLES = 10
42+
NUM_FILES = 120 # enough files to fan out across worker threads (abandoned heaps)
43+
44+
45+
def rss_kb(pid):
46+
out = subprocess.check_output(["ps", "-o", "rss=", "-p", str(pid)])
47+
return int(out.strip())
48+
49+
50+
def make_fixture(d):
51+
for i in range(NUM_FILES):
52+
with open(os.path.join(d, f"mod_{i}.py"), "w") as f:
53+
for j in range(20):
54+
f.write(f"def fn_{i}_{j}(a, b):\n")
55+
f.write(f" x = a + b + {i} * {j}\n")
56+
f.write(" return x\n\n")
57+
58+
59+
def run_series(repo, cache, supervised):
60+
env = dict(os.environ)
61+
env["CBM_CACHE_DIR"] = cache
62+
if supervised:
63+
env.pop("CBM_INDEX_SUPERVISOR", None)
64+
else:
65+
env["CBM_INDEX_SUPERVISOR"] = "0" # in-process (pre-fix background behaviour)
66+
env["CBM_INDEX_WORKER_TIMEOUT_S"] = "120"
67+
68+
proc = subprocess.Popen(
69+
[BINARY], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
70+
stderr=subprocess.DEVNULL, env=env, text=True, bufsize=1,
71+
)
72+
73+
def rpc(obj):
74+
proc.stdin.write(json.dumps(obj) + "\n")
75+
proc.stdin.flush()
76+
return proc.stdout.readline()
77+
78+
rpc({"jsonrpc": "2.0", "id": 0, "method": "initialize", "params": {}})
79+
series = []
80+
for k in range(CYCLES):
81+
rpc({"jsonrpc": "2.0", "id": k + 1, "method": "tools/call",
82+
"params": {"name": "index_repository",
83+
"arguments": {"repo_path": repo, "mode": "fast"}}})
84+
series.append(rss_kb(proc.pid))
85+
try:
86+
proc.stdin.close()
87+
proc.wait(timeout=15)
88+
except Exception:
89+
proc.kill()
90+
return series
91+
92+
93+
def main():
94+
if not os.path.exists(BINARY):
95+
print(f"missing prod binary: {BINARY}\n build it: make -f Makefile.cbm cbm")
96+
return 2
97+
base = tempfile.mkdtemp(prefix="cbm-832-")
98+
repo = os.path.join(base, "repo")
99+
os.makedirs(repo)
100+
make_fixture(repo)
101+
try:
102+
inproc = run_series(repo, os.path.join(base, "c1"), supervised=False)
103+
superv = run_series(repo, os.path.join(base, "c2"), supervised=True)
104+
finally:
105+
shutil.rmtree(base, ignore_errors=True)
106+
107+
def mb(kb):
108+
return kb / 1024.0
109+
110+
print(f"cycles={CYCLES} files={NUM_FILES}")
111+
print("cycle | in-process(MB) | supervised(MB)")
112+
for i in range(CYCLES):
113+
print(f" {i:2d} | {mb(inproc[i]):8.1f} | {mb(superv[i]):8.1f}")
114+
115+
ip_peak = max(mb(x) for x in inproc)
116+
sv_peak = max(mb(x) for x in superv)
117+
print(f"\nin-process peak resident: {ip_peak:8.1f} MB")
118+
print(f"supervised peak resident: {sv_peak:8.1f} MB")
119+
# The decisive, robust signal at laptop-fixture scale is the RESIDENT-LEVEL
120+
# contrast, not cycle-over-cycle growth: the in-process server keeps the whole
121+
# index working set resident (it never leaves the long-lived process), while
122+
# the supervised path returns it every cycle (the child exits) -> the server
123+
# stays near its idle baseline. The unbounded ratchet in the field (#832, GB
124+
# over hours) is the same effect amplified by worker-thread count + cycle count
125+
# beyond what a small fixture surfaces. Generous threshold; report-only,
126+
# NON-GATING.
127+
verdict = "SUPERVISED ISOLATION reproduced (server stays near baseline)" \
128+
if sv_peak < ip_peak / 2 \
129+
else "inconclusive (env-dependent; see numbers)"
130+
print(f"verdict: {verdict}")
131+
return 0
132+
133+
134+
if __name__ == "__main__":
135+
sys.exit(main())

‎tests/test_main.c‎

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,74 @@ int tf_fail_count = 0;
99
int tf_skip_count = 0;
1010

1111
#include "test_framework.h"
12-
#include "foundation/compat.h" /* cbm_setenv — #845 supervisor kill switch */
12+
#include "foundation/compat.h" /* cbm_setenv — #845 supervisor kill switch */
13+
#include "foundation/compat_fs.h" /* cbm_fopen — worker response file */
14+
#include "foundation/mem.h" /* cbm_mem_init — worker budget */
15+
#include "mcp/index_supervisor.h" /* cbm_index_set_worker_role */
16+
#include "mcp/mcp.h" /* cbm_mcp_handle_tool — act as a real worker */
1317
#include <sqlite3.h>
1418
#include <stdbool.h>
19+
#include <stdio.h>
20+
#include <stdlib.h>
1521
#include <string.h>
1622

23+
/* #832 guard support: when the index supervisor spawns THIS binary as
24+
* `<self> cli --index-worker index_repository <args_json> --response-out <file>`
25+
* (exactly the argv cbm_index_spawn_worker builds), act as a faithful in-process
26+
* index worker instead of re-running the test suites. This lets the deterministic
27+
* gating guard (test_mcp.c) spawn a REAL worker child that indexes the fixture and
28+
* writes its response back, using only public APIs — no production test seam.
29+
* Returns an exit code (>=0) when it handled a worker invocation, else -1. */
30+
static int tf_maybe_run_index_worker(int argc, char **argv) {
31+
if (argc < 2 || strcmp(argv[1], "cli") != 0) {
32+
return -1;
33+
}
34+
bool is_worker = false;
35+
const char *tool = NULL;
36+
const char *args_json = "{}";
37+
const char *response_out = NULL;
38+
for (int i = 2; i < argc; i++) {
39+
if (strcmp(argv[i], "--index-worker") == 0) {
40+
is_worker = true;
41+
} else if (strcmp(argv[i], "--response-out") == 0) {
42+
if (i + 1 < argc) {
43+
response_out = argv[++i];
44+
}
45+
} else if (argv[i][0] == '{') {
46+
args_json = argv[i];
47+
} else if (argv[i][0] != '-' && !tool) {
48+
tool = argv[i];
49+
}
50+
}
51+
if (!is_worker) {
52+
return -1;
53+
}
54+
if (!tool) {
55+
tool = "index_repository";
56+
}
57+
58+
cbm_mem_init(0.5);
59+
cbm_index_set_worker_role(true, response_out); /* worker role → index in-process */
60+
cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);
61+
if (!srv) {
62+
return 1;
63+
}
64+
char *result = cbm_mcp_handle_tool(srv, tool, args_json);
65+
if (result) {
66+
const char *ro = cbm_index_worker_response_out();
67+
if (ro) {
68+
FILE *rf = cbm_fopen(ro, "wb");
69+
if (rf) {
70+
(void)fputs(result, rf);
71+
(void)fclose(rf);
72+
}
73+
}
74+
free(result);
75+
}
76+
cbm_mcp_server_free(srv);
77+
return 0;
78+
}
79+
1780
static int g_suite_argc = 0;
1881
static char **g_suite_argv = NULL;
1982

@@ -138,6 +201,13 @@ extern void suite_dump_verify_io(void);
138201
extern void cbm_kind_in_set_free_cache(void);
139202

140203
int main(int argc, char **argv) {
204+
/* #832: if spawned as a supervised index worker, do the real work and exit
205+
* before any suite runs (see tf_maybe_run_index_worker). */
206+
int worker_rc = tf_maybe_run_index_worker(argc, argv);
207+
if (worker_rc >= 0) {
208+
return worker_rc;
209+
}
210+
141211
/* #845 belt-and-suspenders: this binary EMBEDS cbm_mcp_handle_tool. The
142212
* supervisor gate already ignores unmarked hosts, but pin the kill switch
143213
* too so even a future supervisor-marked test host can never resolve THIS

0 commit comments

Comments
 (0)