Skip to content

Commit 19296d7

Browse files
committed
Merge remote-tracking branch 'origin/main' into distill/776-linux-rss-undercount
Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
2 parents 26cd467 + 7884ccb commit 19296d7

11 files changed

Lines changed: 755 additions & 10 deletions

File tree

‎internal/cbm/extract_defs.c‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1869,6 +1869,35 @@ static const char **extract_decorators(CBMArena *a, TSNode node, const char *sou
18691869
* first and one branch is lost (#495). Fold the cfg predicate into the QN so
18701870
* each cfg-gated twin gets a DISTINCT, predicate-encoding QN. Returns the
18711871
* (possibly suffixed) QN; the original QN when no cfg attribute is present. */
1872+
/* Rust: mark a function as a test when it carries a test attribute (#855).
1873+
* cbm's test detection is otherwise file-path-based (cbm_is_test_file:
1874+
* *_test.rs / test_*), so inline #[test]/#[tokio::test] functions inside a
1875+
* regular .rs file are indexed as ordinary Functions (is_test=false) and leak
1876+
* past the store.c `is_test != 1` filter into graph/agent context. The
1877+
* attribute_item text extract_decorators stores is the bracketed form
1878+
* ("#[test]", "#[tokio::test]", "#[tokio::test(...)]", ...). */
1879+
static bool rust_def_is_test(const char *const *decorators) {
1880+
if (!decorators) {
1881+
return false;
1882+
}
1883+
for (int i = 0; decorators[i]; i++) {
1884+
const char *d = decorators[i];
1885+
/* Path-qualified async/param test macros (substring match, robust to the
1886+
* optional argument list and the surrounding #[ ]). */
1887+
if (strstr(d, "tokio::test") || strstr(d, "async_std::test") ||
1888+
strstr(d, "actix_rt::test") || strstr(d, "test_case::case")) {
1889+
return true;
1890+
}
1891+
/* Bare #[test] / #[test(...)]: match the bracketed path exactly so we do
1892+
* NOT match the unrelated #[test_case::case] (handled above) or a
1893+
* hypothetical #[test_crate]. */
1894+
if (strstr(d, "#[test]") || strstr(d, "#[test(")) {
1895+
return true;
1896+
}
1897+
}
1898+
return false;
1899+
}
1900+
18721901
static const char *rust_cfg_qualified_name(CBMArena *a, const char *base_qn,
18731902
const char *const *decorators) {
18741903
if (!decorators) {
@@ -3192,6 +3221,7 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec
31923221
// predicate into the QN so both branches survive the graph upsert (#495).
31933222
if (ctx->language == CBM_LANG_RUST) {
31943223
def.qualified_name = rust_cfg_qualified_name(a, def.qualified_name, def.decorators);
3224+
def.is_test = rust_def_is_test(def.decorators);
31953225
}
31963226

31973227
// Docstring

‎src/foundation/mem.c‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,21 @@ static void check_pressure(size_t rss) {
106106

107107
/* ── Public API ────────────────────────────────────────────────── */
108108

109+
#define RAM_FRACTION_DEFAULT 0.5
110+
#define RAM_FRACTION_16GB 0.25
111+
#define RAM_FRACTION_32GB 0.35
112+
#define RAM_BYTES_PER_GB (1024ULL * 1024 * 1024)
113+
114+
double cbm_mem_ram_fraction_for_total(size_t total_ram_bytes) {
115+
if (total_ram_bytes <= 16ULL * RAM_BYTES_PER_GB) {
116+
return RAM_FRACTION_16GB;
117+
}
118+
if (total_ram_bytes <= 32ULL * RAM_BYTES_PER_GB) {
119+
return RAM_FRACTION_32GB;
120+
}
121+
return RAM_FRACTION_DEFAULT;
122+
}
123+
109124
void cbm_mem_init(double ram_fraction) {
110125
int expected = 0;
111126
if (!atomic_compare_exchange_strong(&g_initialized, &expected, 1)) {
@@ -122,6 +137,15 @@ void cbm_mem_init(double ram_fraction) {
122137
mi_option_set(mi_option_arena_eager_commit, 0);
123138
mi_option_set(mi_option_purge_decommits, SKIP_ONE);
124139
mi_option_set(mi_option_purge_delay, 0); /* immediate purge, no 1s delay */
140+
/* v3 (#832): reclaim abandoned pages on ANY thread's free (=1), restoring the
141+
* v2 behaviour. mimalloc v3 defaults page_reclaim_on_free=0, so pages a worker
142+
* thread abandons at exit are NOT reclaimed when the main thread later frees
143+
* their blocks (and mi_collect cannot touch abandoned pages) — RSS then
144+
* ratchets across repeated in-process index cycles. The supervised subprocess
145+
* is the primary cure (the child returns 100% RSS on exit); this is the
146+
* in-process fallback for any path that stays in-process (kill switch,
147+
* spawn-fail degrade, embedders). */
148+
mi_option_set(mi_option_page_reclaim_on_free, 1);
125149

126150
/* CBM_MEM_BUDGET_MB env override (memory analogue of CBM_WORKERS).
127151
* Lets users cap the budget directly without an enclosing cgroup —

‎src/foundation/mem.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@
1111
#include <stdbool.h>
1212
#include <stddef.h>
1313

14+
/* Tiered default fraction for MCP startup: 25% on <=16GB, 35% on <=32GB, else 50%. */
15+
double cbm_mem_ram_fraction_for_total(size_t total_ram_bytes);
16+
1417
/* Initialize memory budget = ram_fraction * total_physical_ram.
1518
* Thread-safe: only the first call takes effect.
1619
* Configures mimalloc options for reduced upfront memory. */

‎src/main.c‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,6 @@ enum {
3131
MAIN_MAX_PORT = 65536,
3232
PARENT_WATCHDOG_STACK_SIZE = 64 * CBM_SZ_1K, /* watchdog only polls — tiny stack suffices */
3333
};
34-
#define MAIN_RAM_FRACTION 0.5
35-
3634
#define SLEN(s) (sizeof(s) - 1)
3735
#include "foundation/log.h"
3836
#include "foundation/diagnostics.h"
@@ -171,6 +169,23 @@ static int watcher_index_fn(const char *project_name, const char *root_path, voi
171169

172170
cbm_log_info("watcher.reindex", "project", project_name, "path", root_path);
173171

172+
/* #832: route the re-index through the supervised worker subprocess so this
173+
* long-lived server process hands its RSS back to the OS on every cycle
174+
* instead of ratcheting (mimalloc v3 does not reclaim pages that worker
175+
* threads abandon at exit). The child writes the DB; the parent only needs the
176+
* return code. The pipeline lock (already held) still serialises re-indexes.
177+
* Degrade to the in-process pipeline when the supervisor is off (kill switch)
178+
* or the spawn fails. */
179+
if (cbm_index_supervisor_should_wrap()) {
180+
char *resp = cbm_mcp_index_run_supervised_path(root_path);
181+
if (resp) {
182+
free(resp);
183+
cbm_pipeline_unlock();
184+
return 0;
185+
}
186+
/* resp == NULL → spawn-failure degrade → fall through to in-process. */
187+
}
188+
174189
cbm_pipeline_t *p = cbm_pipeline_new(root_path, NULL, CBM_MODE_FULL);
175190
if (!p) {
176191
cbm_pipeline_unlock();
@@ -518,11 +533,11 @@ static int handle_subcommand(int argc, char **argv) {
518533
return 0;
519534
}
520535
if (strcmp(argv[i], "cli") == 0) {
521-
cbm_mem_init(MAIN_RAM_FRACTION);
536+
cbm_mem_init(cbm_mem_ram_fraction_for_total(cbm_system_info().total_ram));
522537
return run_cli(argc - i - SKIP_ONE, argv + i + SKIP_ONE);
523538
}
524539
if (strcmp(argv[i], "hook-augment") == 0) {
525-
cbm_mem_init(MAIN_RAM_FRACTION);
540+
cbm_mem_init(cbm_mem_ram_fraction_for_total(cbm_system_info().total_ram));
526541
return cbm_cmd_hook_augment();
527542
}
528543
if (strcmp(argv[i], "install") == 0) {
@@ -626,9 +641,7 @@ int main(int argc, char **argv) {
626641
#endif
627642

628643
/* Default: MCP server on stdio */
629-
cbm_mem_init(MAIN_RAM_FRACTION); /* 50% of RAM — safe now because mimalloc tracks ALL
630-
* memory (C + C++ allocations) via global override.
631-
* No more untracked heap blind spots. */
644+
cbm_mem_init(cbm_mem_ram_fraction_for_total(cbm_system_info().total_ram));
632645
/* Store binary path for subprocess spawning + hook log sink */
633646
cbm_http_server_set_binary_path(argv[0]);
634647
cbm_log_set_sink_ex(cbm_ui_log_append, CBM_LOG_SINK_TEE);

‎src/mcp/mcp.c‎

Lines changed: 56 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3481,8 +3481,14 @@ static char *build_worker_failure_response(const char *args, cbm_proc_outcome_t
34813481
}
34823482

34833483
/* Drop the cached store so the next query reopens whatever the worker wrote (each
3484-
* worker is a fresh process that deletes + recreates the .db). */
3484+
* worker is a fresh process that deletes + recreates the .db). NULL-safe: the
3485+
* background watcher path (main.c) has no MCP server / cached store — the child
3486+
* writes the DB and the parent only needs the return code, so there is nothing
3487+
* to invalidate. */
34853488
static void supervisor_invalidate_store(cbm_mcp_server_t *srv) {
3489+
if (!srv) {
3490+
return;
3491+
}
34863492
if (srv->owns_store && srv->store) {
34873493
cbm_store_close(srv->store);
34883494
srv->store = NULL;
@@ -3687,6 +3693,34 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) {
36873693
return build_worker_failure_response(args, last_outcome);
36883694
}
36893695

3696+
/* Build a minimal {"repo_path": "<root>"} args object (path safely escaped) and
3697+
* run it through index_run_supervised. Shared by the session auto-index (srv
3698+
* present → its cached store is invalidated) and the watcher re-index (srv NULL).
3699+
* Returns the worker's response string (caller frees) or NULL to degrade. */
3700+
static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_path) {
3701+
if (!root_path || !root_path[0]) {
3702+
return NULL;
3703+
}
3704+
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
3705+
yyjson_mut_val *root = yyjson_mut_obj(doc);
3706+
yyjson_mut_doc_set_root(doc, root);
3707+
yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path);
3708+
char *args = yy_doc_to_str(doc);
3709+
yyjson_mut_doc_free(doc);
3710+
if (!args) {
3711+
return NULL;
3712+
}
3713+
char *resp = index_run_supervised(srv, args);
3714+
free(args);
3715+
return resp;
3716+
}
3717+
3718+
/* Public entry (see mcp.h): the watcher re-index in main.c has no MCP server, so
3719+
* it reaches the supervised runner through this srv-less wrapper. */
3720+
char *cbm_mcp_index_run_supervised_path(const char *root_path) {
3721+
return index_run_supervised_path(NULL, root_path);
3722+
}
3723+
36903724
static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
36913725
/* Supervisor gate: run the index in a crash/hang-isolating worker subprocess
36923726
* unless this process IS the worker or the kill switch (CBM_INDEX_SUPERVISOR=0)
@@ -5662,6 +5696,26 @@ static void *autoindex_thread(void *arg) {
56625696

56635697
cbm_log_info("autoindex.start", "project", srv->session_project, "path", srv->session_root);
56645698

5699+
/* #832: prefer the supervised worker subprocess. Indexing the whole session in
5700+
* this long-lived server thread ratchets RSS (mimalloc v3 does not reclaim the
5701+
* pages worker threads abandon at exit); running it in a child that exits hands
5702+
* 100% of that memory back to the OS every cycle. Degrade to the in-process
5703+
* pipeline below when the supervisor is off (kill switch) or the spawn fails. */
5704+
if (cbm_index_supervisor_should_wrap()) {
5705+
char *resp = index_run_supervised_path(srv, srv->session_root);
5706+
if (resp) {
5707+
free(resp);
5708+
cbm_log_info("autoindex.done", "project", srv->session_project, "mode", "supervised");
5709+
/* Register with watcher for ongoing change detection — gated on
5710+
* auto_watch (#849), same as the in-process branch below. A bare
5711+
* `if (srv->watcher)` would register even when the user set
5712+
* `config set auto_watch false`, since srv->watcher is always set. */
5713+
register_watcher_if_enabled(srv);
5714+
return NULL;
5715+
}
5716+
/* resp == NULL → spawn-failure degrade → fall through to in-process. */
5717+
}
5718+
56655719
cbm_pipeline_t *p = cbm_pipeline_new(srv->session_root, NULL, CBM_MODE_FULL);
56665720
if (!p) {
56675721
cbm_log_warn("autoindex.err", "msg", "pipeline_create_failed");
@@ -5674,7 +5728,7 @@ static void *autoindex_thread(void *arg) {
56745728
cbm_pipeline_unlock();
56755729

56765730
cbm_pipeline_free(p);
5677-
cbm_mem_collect(); /* return mimalloc pages to OS after indexing */
5731+
cbm_mem_collect(); /* return mimalloc pages to OS after indexing (in-process only) */
56785732

56795733
if (rc == 0) {
56805734
cbm_log_info("autoindex.done", "project", srv->session_project);

‎src/mcp/mcp.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,18 @@ char *cbm_mcp_server_handle(cbm_mcp_server_t *srv, const char *line);
116116
/* Handle a tools/call request. Returns MCP tool result JSON. */
117117
char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const char *args_json);
118118

119+
/* ── Supervised background index (RSS isolation, #832) ────────── */
120+
121+
/* Run a full index of root_path in a supervised worker SUBPROCESS (the same
122+
* crash/hang-isolating runner used by handle_index_repository), so the child
123+
* returns 100% of its RSS to the OS on exit instead of ratcheting the long-lived
124+
* parent. Builds {"repo_path": root_path} internally. Returns the worker's
125+
* response string (caller frees) on success, or NULL to signal the caller must
126+
* degrade to the in-process path (kill switch set, spawn failure, or the process
127+
* is not a supervisor host). This is the shared entry the watcher re-index
128+
* (main.c) and the session auto-index (mcp.c) route through. */
129+
char *cbm_mcp_index_run_supervised_path(const char *root_path);
130+
119131
/* ── Idle store eviction ──────────────────────────────────────── */
120132

121133
/* Evict the cached project store if idle for more than timeout_s seconds.

‎tests/repro/issue832_rss.py‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
#!/usr/bin/env python3
2+
"""
3+
issue832_rss.py -- direct RSS reproduction for #832 (NON-GATING, manual repro tier).
4+
5+
WHY THIS IS NOT A C UNIT TEST
6+
-----------------------------
7+
The RSS ratchet is a property of mimalloc v3's abandoned-page handling
8+
(page_reclaim_on_free=0): pages a worker THREAD abandons at exit are not
9+
reclaimed when the main thread later frees their blocks. That only manifests in
10+
the PROD binary, which links mimalloc as the global allocator (Makefile.cbm:
11+
MI_OVERRIDE=1). The C test-runner and the C repro-runner are built CRT+ASan
12+
(MI_OVERRIDE=0), so mimalloc is inert there and cbm_mem_rss() falls back to
13+
os_rss() -- a C test would be VACUOUS. Hence this drives the real
14+
`build/c/codebase-memory-mcp` server over stdio and samples its RSS from `ps`.
15+
16+
WHAT IT SHOWS
17+
-------------
18+
A long-lived MCP server is driven through K index_repository cycles of the same
19+
fixture. The in-process pipeline (CBM_INDEX_SUPERVISOR=0) is the pre-#832-fix
20+
background-path behaviour: RSS RATCHETS across cycles. The supervised subprocess
21+
path (default) is the fix: each child returns 100% of its RSS on exit, so the
22+
long-lived parent stays ~FLAT. The auto-index (mcp.c) and watcher re-index
23+
(main.c) paths now route through that same supervised subprocess, so they inherit
24+
this flat profile; the deterministic routing proof is the GATING guard
25+
tests/test_mcp.c::index_bg_paths_route_through_supervisor_issue832.
26+
27+
Inherently noisy (allocator/OS dependent) -> thresholds are generous and this is
28+
NOT wired into `make test` / `ci-ok`. Run manually:
29+
make -f Makefile.cbm cbm
30+
python3 tests/repro/issue832_rss.py
31+
"""
32+
import json
33+
import os
34+
import shutil
35+
import subprocess
36+
import sys
37+
import tempfile
38+
39+
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
40+
BINARY = os.path.join(ROOT, "build", "c", "codebase-memory-mcp")
41+
CYCLES = 10
42+
NUM_FILES = 120 # enough files to fan out across worker threads (abandoned heaps)
43+
44+
45+
def rss_kb(pid):
46+
out = subprocess.check_output(["ps", "-o", "rss=", "-p", str(pid)])
47+
return int(out.strip())
48+
49+
50+
def make_fixture(d):
51+
for i in range(NUM_FILES):
52+
with open(os.path.join(d, f"mod_{i}.py"), "w") as f:
53+
for j in range(20):
54+
f.write(f"def fn_{i}_{j}(a, b):\n")
55+
f.write(f" x = a + b + {i} * {j}\n")
56+
f.write(" return x\n\n")
57+
58+
59+
def run_series(repo, cache, supervised):
60+
env = dict(os.environ)
61+
env["CBM_CACHE_DIR"] = cache
62+
if supervised:
63+
env.pop("CBM_INDEX_SUPERVISOR", None)
64+
else:
65+
env["CBM_INDEX_SUPERVISOR"] = "0" # in-process (pre-fix background behaviour)
66+
env["CBM_INDEX_WORKER_TIMEOUT_S"] = "120"
67+
68+
proc = subprocess.Popen(
69+
[BINARY], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
70+
stderr=subprocess.DEVNULL, env=env, text=True, bufsize=1,
71+
)
72+
73+
def rpc(obj):
74+
proc.stdin.write(json.dumps(obj) + "\n")
75+
proc.stdin.flush()
76+
return proc.stdout.readline()
77+
78+
rpc({"jsonrpc": "2.0", "id": 0, "method": "initialize", "params": {}})
79+
series = []
80+
for k in range(CYCLES):
81+
rpc({"jsonrpc": "2.0", "id": k + 1, "method": "tools/call",
82+
"params": {"name": "index_repository",
83+
"arguments": {"repo_path": repo, "mode": "fast"}}})
84+
series.append(rss_kb(proc.pid))
85+
try:
86+
proc.stdin.close()
87+
proc.wait(timeout=15)
88+
except Exception:
89+
proc.kill()
90+
return series
91+
92+
93+
def main():
94+
if not os.path.exists(BINARY):
95+
print(f"missing prod binary: {BINARY}\n build it: make -f Makefile.cbm cbm")
96+
return 2
97+
base = tempfile.mkdtemp(prefix="cbm-832-")
98+
repo = os.path.join(base, "repo")
99+
os.makedirs(repo)
100+
make_fixture(repo)
101+
try:
102+
inproc = run_series(repo, os.path.join(base, "c1"), supervised=False)
103+
superv = run_series(repo, os.path.join(base, "c2"), supervised=True)
104+
finally:
105+
shutil.rmtree(base, ignore_errors=True)
106+
107+
def mb(kb):
108+
return kb / 1024.0
109+
110+
print(f"cycles={CYCLES} files={NUM_FILES}")
111+
print("cycle | in-process(MB) | supervised(MB)")
112+
for i in range(CYCLES):
113+
print(f" {i:2d} | {mb(inproc[i]):8.1f} | {mb(superv[i]):8.1f}")
114+
115+
ip_peak = max(mb(x) for x in inproc)
116+
sv_peak = max(mb(x) for x in superv)
117+
print(f"\nin-process peak resident: {ip_peak:8.1f} MB")
118+
print(f"supervised peak resident: {sv_peak:8.1f} MB")
119+
# The decisive, robust signal at laptop-fixture scale is the RESIDENT-LEVEL
120+
# contrast, not cycle-over-cycle growth: the in-process server keeps the whole
121+
# index working set resident (it never leaves the long-lived process), while
122+
# the supervised path returns it every cycle (the child exits) -> the server
123+
# stays near its idle baseline. The unbounded ratchet in the field (#832, GB
124+
# over hours) is the same effect amplified by worker-thread count + cycle count
125+
# beyond what a small fixture surfaces. Generous threshold; report-only,
126+
# NON-GATING.
127+
verdict = "SUPERVISED ISOLATION reproduced (server stays near baseline)" \
128+
if sv_peak < ip_peak / 2 \
129+
else "inconclusive (env-dependent; see numbers)"
130+
print(f"verdict: {verdict}")
131+
return 0
132+
133+
134+
if __name__ == "__main__":
135+
sys.exit(main())

0 commit comments

Comments
 (0)