Skip to content

Commit acee801

Browse files
authored
Merge pull request DeusData#675 from DeusData/fix/581-leak-mitigation
feat(diag): persistent memory trajectory + query-leak CI soak (DeusData#581 observability)
2 parents 4ab00df + f7d6d34 commit acee801

5 files changed

Lines changed: 123 additions & 4 deletions

File tree

‎.github/ISSUE_TEMPLATE/bug_report.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,21 @@ body:
103103
Please check for private paths/strings before pasting.
104104
render: text
105105

106+
- type: textarea
107+
id: diagnostics
108+
attributes:
109+
label: Diagnostics trajectory (memory / performance / leak issues)
110+
description: |
111+
We collect **no telemetry**, so for a memory or performance problem we
112+
need the diagnostics trajectory from your machine. Set
113+
`CBM_DIAGNOSTICS=1`, reproduce the issue, then attach (or paste) the file
114+
`cbm-diagnostics-<pid>.ndjson` from your temp dir (`$TMPDIR` / `/tmp`, or
115+
`%TEMP%` on Windows). It records only resource counters (rss, committed,
116+
fds, query counts) over time — no code or queries. See the README
117+
"Troubleshooting & Diagnostics" section. Pasting an agent's summary of
118+
the trajectory is also fine.
119+
render: text
120+
106121
- type: input
107122
id: scale
108123
attributes:

‎.github/workflows/_soak.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,13 +62,27 @@ jobs:
6262
run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=${{ matrix.cc }} CXX=${{ matrix.cxx }}
6363
- name: Soak (${{ inputs.duration_minutes }} min)
6464
run: scripts/soak-test.sh build/c/codebase-memory-mcp ${{ inputs.duration_minutes }}
65+
# #581 guard: read-only soak (never reindex/mutate) so any memory growth is
66+
# a query-path leak, not WAL/indexing. Reuses the build above.
67+
- name: Query-leak soak (#581, read-only)
68+
env:
69+
CBM_SOAK_MODE: query-leak
70+
RESULTS_DIR: soak-results-query-leak
71+
run: scripts/soak-test.sh build/c/codebase-memory-mcp ${{ inputs.duration_minutes }} --skip-crash-test
6572
- name: Upload metrics
6673
if: always()
6774
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
6875
with:
6976
name: soak-quick-${{ matrix.goos }}-${{ matrix.goarch }}
7077
path: soak-results/
7178
retention-days: 14
79+
- name: Upload query-leak metrics
80+
if: always()
81+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
82+
with:
83+
name: soak-query-leak-${{ matrix.goos }}-${{ matrix.goarch }}
84+
path: soak-results-query-leak/
85+
retention-days: 14
7286

7387
soak-quick-windows:
7488
runs-on: windows-latest
@@ -98,13 +112,30 @@ jobs:
98112
BIN=build/c/codebase-memory-mcp
99113
[ -f "${BIN}.exe" ] && BIN="${BIN}.exe"
100114
scripts/soak-test.sh "$BIN" ${{ inputs.duration_minutes }}
115+
# #581 guard on Windows — the platform the bug is reported on.
116+
- name: Query-leak soak (#581, read-only)
117+
shell: msys2 {0}
118+
env:
119+
CBM_SOAK_MODE: query-leak
120+
RESULTS_DIR: soak-results-query-leak
121+
run: |
122+
BIN=build/c/codebase-memory-mcp
123+
[ -f "${BIN}.exe" ] && BIN="${BIN}.exe"
124+
scripts/soak-test.sh "$BIN" ${{ inputs.duration_minutes }} --skip-crash-test
101125
- name: Upload metrics
102126
if: always()
103127
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
104128
with:
105129
name: soak-quick-windows-amd64
106130
path: soak-results/
107131
retention-days: 14
132+
- name: Upload query-leak metrics
133+
if: always()
134+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
135+
with:
136+
name: soak-query-leak-windows-amd64
137+
path: soak-results-query-leak/
138+
retention-days: 14
108139

109140
soak-asan:
110141
if: ${{ inputs.run_asan }}

‎README.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,31 @@ Benchmarked on Apple M3 Pro:
231231

232232
**Token efficiency**: Five structural queries consumed ~3,400 tokens via codebase-memory-mcp versus ~412,000 tokens via file-by-file grep exploration — a **99.2% reduction**.
233233

234+
## Troubleshooting & Diagnostics
235+
236+
codebase-memory-mcp runs **100% locally and collects no telemetry** — your code, queries, environment, and usage never leave your machine. That privacy guarantee also means that when you hit something we can't reproduce on our side (a slow memory climb over hours, a performance regression, a leak that only appears after days of real use), **we have no data at all unless you choose to send it.** Here is how to capture it yourself.
237+
238+
### Capture a diagnostics log
239+
240+
Set `CBM_DIAGNOSTICS=1` before the MCP server starts, then reproduce the problem (let it run as long as it takes — a slow leak needs time to show in the trend). The server writes two files to your system temp directory (`$TMPDIR` or `/tmp` on macOS/Linux, `%TEMP%` on Windows):
241+
242+
| File | What it is |
243+
|------|------------|
244+
| `cbm-diagnostics-<pid>.ndjson` | **The memory trajectory** — one JSON line every 5 s with `rss`, `committed` (Windows commit charge), `peak_*`, `page_faults`, `fd`, and `queries`. **This is the file we need for memory/leak reports** — the *trend over time* is what pinpoints a leak. It is **kept on disk after the server exits** (so you can grab it post-mortem) and rotates to `.ndjson.1` past ~8 MB. |
245+
| `cbm-diagnostics-<pid>.json` | The latest snapshot only — handy for a quick live check. Removed on clean exit. |
246+
247+
The startup log prints both paths, e.g.:
248+
249+
```
250+
level=info msg=diagnostics.start snapshot=/tmp/cbm-diagnostics-12345.json trajectory=/tmp/cbm-diagnostics-12345.ndjson interval=5s
251+
```
252+
253+
Set the variable in the `env` block of your agent's MCP server config, or export it before launching the server.
254+
255+
### What to share
256+
257+
When you open a memory/performance issue, **attach the `.ndjson` trajectory** — it contains no source code or query text, only resource counters. If you'd rather not attach a file, paste it (or an agent's summary of it) into the issue: your assistant can read the NDJSON directly and report whether `rss`/`committed` grow monotonically, how fast, and relative to query count — which is exactly what we need to find the cause.
258+
234259
## Installation
235260

236261
### Pre-built Binaries

‎scripts/soak-test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ BINARY=$(cd "$(dirname "$BINARY")" && pwd)/$(basename "$BINARY")
3434
# skipped in this mode because it reindexes (which would mask #581).
3535
CBM_SOAK_MODE="${CBM_SOAK_MODE:-default}"
3636

37-
RESULTS_DIR="soak-results"
37+
RESULTS_DIR="${RESULTS_DIR:-soak-results}"
3838
mkdir -p "$RESULTS_DIR"
3939

4040
METRICS_CSV="$RESULTS_DIR/metrics.csv"

‎src/foundation/diagnostics.c‎

Lines changed: 51 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,12 @@ static cbm_thread_t g_diag_thread;
3535
static bool g_diag_started = false;
3636
static time_t g_start_time = 0;
3737
static char g_diag_path[CBM_SZ_256] = "";
38+
/* Persistent NDJSON time-series (the memory TRAJECTORY users send us to diagnose
39+
* slow leaks like #581). Unlike g_diag_path (a latest-snapshot file overwritten
40+
* every interval and deleted on stop), this is appended-to and KEPT on exit. */
41+
static char g_diag_ndjson_path[CBM_SZ_256] = "";
42+
static size_t g_diag_ndjson_size = 0;
43+
#define DIAG_NDJSON_CAP_BYTES (8u * 1024u * 1024u) /* rotate to .1 past this */
3844

3945
/* ── Query stats ─────────────────────────────────────────────────── */
4046

@@ -89,6 +95,35 @@ static int count_open_fds(void) {
8995
#define DIAG_INTERVAL_S 5
9096
#define DIAG_PATH_EXTRA 24 /* ".tmp" + safety margin */
9197

98+
/* Append one compact JSON line to the persistent NDJSON trajectory, rotating to
99+
* <path>.1 once it passes the cap. Best-effort: a failed append never disrupts
100+
* the server. The trajectory (a monotonic rss/committed climb over hours) is
101+
* what reveals a slow leak like #581 — the single latest-snapshot file cannot. */
102+
static void append_trajectory(long uptime, size_t rss, size_t peak_rss, size_t commit,
103+
size_t peak_commit, size_t page_faults, int fds, int qcount) {
104+
if (g_diag_ndjson_path[0] == '\0') {
105+
return;
106+
}
107+
if (g_diag_ndjson_size > DIAG_NDJSON_CAP_BYTES) {
108+
char rot[sizeof(g_diag_ndjson_path) + DIAG_PATH_EXTRA];
109+
snprintf(rot, sizeof(rot), "%s.1", g_diag_ndjson_path);
110+
(void)rename(g_diag_ndjson_path, rot); /* keep one previous generation */
111+
g_diag_ndjson_size = 0;
112+
}
113+
FILE *f = fopen(g_diag_ndjson_path, "a");
114+
if (!f) {
115+
return;
116+
}
117+
int n = fprintf(f,
118+
"{\"uptime_s\":%ld,\"rss\":%zu,\"peak_rss\":%zu,\"committed\":%zu,"
119+
"\"peak_committed\":%zu,\"page_faults\":%zu,\"fd\":%d,\"queries\":%d}\n",
120+
uptime, rss, peak_rss, commit, peak_commit, page_faults, fds, qcount);
121+
if (n > 0) {
122+
g_diag_ndjson_size += (size_t)n;
123+
}
124+
(void)fclose(f);
125+
}
126+
92127
static void write_diagnostics(void) {
93128
/* Collect mimalloc stats */
94129
size_t elapsed_ms = 0;
@@ -151,6 +186,10 @@ static void write_diagnostics(void) {
151186
return;
152187
}
153188
(void)rename(tmp_path, g_diag_path);
189+
190+
/* Also append to the persistent trajectory (kept on exit for users to send). */
191+
append_trajectory(uptime, current_rss, peak_rss, current_commit, peak_commit, page_faults, fds,
192+
qcount);
154193
}
155194

156195
static void *diag_thread_fn(void *arg) {
@@ -179,14 +218,18 @@ bool cbm_diag_start(void) {
179218

180219
snprintf(g_diag_path, sizeof(g_diag_path), "%s/cbm-diagnostics-%d.json", cbm_tmpdir(),
181220
(int)getpid());
221+
snprintf(g_diag_ndjson_path, sizeof(g_diag_ndjson_path), "%s/cbm-diagnostics-%d.ndjson",
222+
cbm_tmpdir(), (int)getpid());
223+
g_diag_ndjson_size = 0;
182224

183225
if (cbm_thread_create(&g_diag_thread, 0, diag_thread_fn, NULL) != 0) {
184226
return false;
185227
}
186228

187229
g_diag_started = true;
188-
(void)fprintf(stderr, "level=info msg=diagnostics.start path=%s interval=%ds\n", g_diag_path,
189-
DIAG_INTERVAL_S);
230+
(void)fprintf(stderr,
231+
"level=info msg=diagnostics.start snapshot=%s trajectory=%s interval=%ds\n",
232+
g_diag_path, g_diag_ndjson_path, DIAG_INTERVAL_S);
190233
return true;
191234
}
192235

@@ -198,9 +241,14 @@ void cbm_diag_stop(void) {
198241
cbm_thread_join(&g_diag_thread);
199242
g_diag_started = false;
200243

201-
/* Clean up file */
244+
/* Remove the live latest-snapshot file (and its .tmp), but KEEP the
245+
* trajectory NDJSON so the user can send it after the server exits. */
202246
cbm_unlink(g_diag_path);
203247
char tmp_path[sizeof(g_diag_path) + DIAG_PATH_EXTRA];
204248
snprintf(tmp_path, sizeof(tmp_path), "%s.tmp", g_diag_path);
205249
cbm_unlink(tmp_path);
250+
if (g_diag_ndjson_path[0] != '\0') {
251+
(void)fprintf(stderr, "level=info msg=diagnostics.trajectory_kept path=%s\n",
252+
g_diag_ndjson_path);
253+
}
206254
}

0 commit comments

Comments
 (0)