Skip to content

Commit 334a408

Browse files
authored
Merge pull request DeusData#2198 from astandrik/fix/1696-bench-runtime-isolation
fix(test): isolate benchmark harness daemon runtimes
2 parents b461b91 + a7a9f27 commit 334a408

5 files changed

Lines changed: 483 additions & 15 deletions

File tree

‎docs/EVALUATION_PLAN.md‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -274,17 +274,21 @@ then its index is **deleted** before the next language — so **only one project
274274
every index is genuinely cold. The skeleton:
275275

276276
```bash
277-
# Start clean: no leftover DBs, fresh results tree
278-
rm -f ~/.cache/codebase-memory-mcp/*.db
277+
# Start clean: fresh results tree
279278
mkdir -p /tmp/eval-results
280279

281280
for lang in $ALL_LANGS; do # ALL_LANGS = full 159-name list (§2 / §8)
282281
manifest_done "$lang" && continue # [CR-8] skip already-completed languages
283282

284283
# --- step 2: cold index in the main channel, TIMED (key metric) ---
285284
t0=$(now_ms)
285+
CBM_BENCH_KEEP_RUNTIME=1 \
286286
scripts/benchmark-index.sh ~/.local/bin/codebase-memory-mcp "$lang" /tmp/bench/"$lang" /tmp/eval-results
287287
index_ms=$(( $(now_ms) - t0 )) # clone+index wall-clock → manifest + report (§5)
288+
# The harness indexes into a run-private root (#1696). CBM_BENCH_KEEP_RUNTIME leaves that
289+
# root behind on success and records its paths; the graph session in steps 4-7 must be
290+
# started with these CBM_RUNTIME_DIR / CBM_CACHE_DIR, and step 8 removes the root.
291+
set -a; . /tmp/eval-results/"$lang"/runtime-root.txt; set +a
288292

289293
# --- step 3: record per-type histograms (zeros back-filled) ---
290294
# node-types.json, edge-types.json (every label + all 32 edge types, zeros kept, §7 below)
@@ -293,7 +297,14 @@ for lang in $ALL_LANGS; do # ALL_LANGS = full 159-name list
293297
# + per-language report + (deferred, blind) judge ---
294298

295299
# --- step 8: delete THIS language's index so the next is cold, then mark done ---
296-
rm -f ~/.cache/codebase-memory-mcp/*.db
300+
# The graph session (steps 4-7) may still have a daemon running under the kept
301+
# CBM_RUNTIME_DIR (exported above); stop it before its root goes away. A refused
302+
# stop (a session still attached) means the root is still in use: keep it.
303+
if ~/.local/bin/codebase-memory-mcp daemon stop; then
304+
rm -rf -- "$CBM_BENCH_RUNTIME_ROOT"
305+
else
306+
echo "$lang: daemon under $CBM_RUNTIME_DIR did not stop; keeping $CBM_BENCH_RUNTIME_ROOT" >&2
307+
fi
297308
manifest_mark_done "$lang" "$index_ms"
298309
done
299310
```
@@ -860,10 +871,13 @@ Deep-Dive section.
860871
# 1. Clone all 159 repos (shallow; skip existing)
861872
scripts/clone-bench-repos.sh /tmp/bench
862873

863-
# 2. Cold index all 159 (LSP cohort in full mode)
864-
rm -f ~/.cache/codebase-memory-mcp/*.db
874+
# 2. Cold index all 159 (LSP cohort in full mode). Each language is indexed into a run-private
875+
# runtime and cache (#1696); CBM_BENCH_KEEP_RUNTIME=1 keeps that root after a successful
876+
# index and records its paths in /tmp/eval-results/<lang>/runtime-root.txt. Source it
877+
# before the graph phase, then stop the daemon and remove the root, as in §7 step 8.
865878
mkdir -p /tmp/eval-results
866879
for lang in $ALL_LANGS; do
880+
CBM_BENCH_KEEP_RUNTIME=1 \
867881
scripts/benchmark-index.sh ~/.local/bin/codebase-memory-mcp "$lang" /tmp/bench/"$lang" /tmp/eval-results
868882
done
869883

‎scripts/benchmark-index.sh‎

Lines changed: 106 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,74 @@ LANG="${2:?}"
99
REPO="${3:?}"
1010
RESULTS_DIR="${4:?}"
1111

12+
OUT="$RESULTS_DIR/$LANG"
13+
mkdir -p "$OUT"
14+
# A handoff names this run's index or none: the evaluation loop reuses the
15+
# results directory, and a stale one would point the graph session at a root
16+
# step 8 has already removed. It goes before any setup step that can fail, so
17+
# a run that never reaches the index cannot leave an earlier one behind.
18+
rm -f -- "$OUT/runtime-root.txt"
19+
20+
# The index must run against a daemon rendezvous and cache this run owns: only
21+
# CBM_RUNTIME_DIR moves the rendezvous, and without a private cache the
22+
# benchmark repository was indexed into the operator's live store (#1696).
23+
# shellcheck source=test-runtime.sh
24+
source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"
25+
cbm_test_runtime_init
26+
27+
# The evaluation plan (docs/EVALUATION_PLAN.md §7) indexes a language here and
28+
# then answers graph questions against that index from its own MCP session.
29+
# A run-private root would be gone before that session starts, so the caller
30+
# may ask for it to be kept: after a SUCCESSFUL run the harness stops its
31+
# daemon, leaves the root in place, records the paths that reach it in
32+
# <results>/<lang>/runtime-root.txt (sourceable), and ownership of the root —
33+
# including its removal — passes to the caller. A failed run cleans up
34+
# regardless: there is no index worth keeping, and nothing must leak. Never on
35+
# by default, or an unattended run accumulates one root per language.
36+
#
37+
# The handoff promises a stopped daemon, so it is written only once
38+
# `daemon stop` succeeded. A refused stop (a client still committed to the
39+
# daemon, or no answer) fails the run with the product's explanation, writes
40+
# no handoff, and keeps the root without cleaning it up: the daemon may still
41+
# be using it, and the operator decides what happens to it.
42+
bench_finish() {
43+
local rc=$? stop_out=""
44+
[ -z "${BENCH_TMP:-}" ] || rm -rf -- "$BENCH_TMP" || true
45+
if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then
46+
if ! stop_out=$("$BINARY" daemon stop 2>&1); then
47+
echo " $LANG: private daemon did not stop; runtime kept at $CBM_TEST_RUNTIME_ROOT, no handoff written" >&2
48+
[ -z "$stop_out" ] || printf '%s\n' "$stop_out" >&2
49+
exit 1
50+
fi
51+
printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \
52+
"$CBM_TEST_RUNTIME_ROOT" "$CBM_RUNTIME_DIR" "$CBM_CACHE_DIR" > "$OUT/runtime-root.txt"
53+
echo " $LANG: runtime kept at $CBM_TEST_RUNTIME_ROOT; paths in $OUT/runtime-root.txt" >&2
54+
return 0
55+
fi
56+
cbm_test_runtime_cleanup "$BINARY"
57+
}
58+
BENCH_TMP=""
59+
trap bench_finish EXIT
60+
# The index call keeps its stderr, the way the search-graph twin does: for a
61+
# one-shot CLI that is the only channel a refusal is reported on, and without
62+
# it the failure below could name only its symptom.
63+
BENCH_TMP=$(mktemp -d)
64+
INDEX_ERR="$BENCH_TMP/index-stderr.log"
65+
DAEMON_START_LOG="$BENCH_TMP/daemon-start.log"
66+
1267
# Resolve symlinks
1368
REPO=$(cd "$REPO" && pwd -P)
69+
# One pre-escaped spelling of the path for every request below, the way the
70+
# soak harness builds its own: a repository path may legitimately contain a
71+
# quote or a backslash, and hand-built JSON turns that into a parse error.
72+
REPO_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$REPO")
1473

15-
OUT="$RESULTS_DIR/$LANG"
16-
mkdir -p "$OUT"
74+
# Elapsed time is read from a monotonic clock, never the wall clock: an NTP
75+
# step mid-run would otherwise skew — or negate — a figure whose whole purpose
76+
# is comparison across runs. Its reference point is fixed per boot on every
77+
# platform CPython supports here, so the three readings below are comparable
78+
# even though each comes from its own process.
79+
bench_now_ms() { python3 -c "import time; print(time.monotonic_ns() // 1000000)"; }
1780

1881
echo "INDEX: $LANG ($REPO)"
1982

@@ -33,16 +96,39 @@ LOC=$(find "$REPO" -type f \
3396
echo "$FILE_COUNT" > "$OUT/file-count.txt"
3497
echo "$LOC" > "$OUT/loc.txt"
3598

99+
# Start the private daemon before timing so index-time.txt measures the index
100+
# alone. setup-time.txt keeps the activation cost attributable and
101+
# total-time.txt is their sum — the figure comparable with earlier runs, which
102+
# paid activation inside the index timing whenever no daemon was already warm.
103+
SETUP_START_MS=$(bench_now_ms)
104+
# Its output is kept for the failure path: the product names why a daemon did
105+
# not start, and that is the part an operator needs.
106+
if ! "$BINARY" daemon start >"$DAEMON_START_LOG" 2>&1; then
107+
echo " $LANG: private daemon did not start" >&2
108+
if [ -s "$DAEMON_START_LOG" ]; then
109+
echo "--- daemon start output ---" >&2
110+
cat "$DAEMON_START_LOG" >&2
111+
fi
112+
exit 1
113+
fi
114+
36115
# Index via CLI and capture timing
37-
START_MS=$(python3 -c "import time; print(int(time.time()*1000))")
116+
START_MS=$(bench_now_ms)
38117

39-
INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}')
118+
# The CLI's exit status is the index's verdict. It is recorded here and acted
119+
# on below, once every per-run file is written; an empty response stays valid
120+
# JSON in 00-index.json so the failure is legible there as well.
121+
INDEX_RC=0
122+
INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>"$INDEX_ERR") ||
123+
{ INDEX_RC=$?; INDEX_JSON="{\"error\":\"index failed\",\"exit\":$INDEX_RC}"; }
40124

41-
END_MS=$(python3 -c "import time; print(int(time.time()*1000))")
125+
END_MS=$(bench_now_ms)
42126
ELAPSED=$((END_MS - START_MS))
43127

44128
echo "$INDEX_JSON" > "$OUT/00-index.json"
45129
echo "$ELAPSED" > "$OUT/index-time.txt"
130+
echo "$((START_MS - SETUP_START_MS))" > "$OUT/setup-time.txt"
131+
echo "$((END_MS - SETUP_START_MS))" > "$OUT/total-time.txt"
46132

47133
# Extract node/edge counts (CLI wraps in MCP content envelope)
48134
NODES=$(echo "$INDEX_JSON" | python3 -c "
@@ -78,5 +164,20 @@ echo "$NODES" > "$OUT/nodes.txt"
78164
echo "$EDGES" > "$OUT/edges.txt"
79165
echo "$PROJECT" > "$OUT/project.txt"
80166

167+
# An index that failed, or that named no project, fails the run: after the
168+
# timing files, so the caller keeps its figures, and through the exit status,
169+
# so the caller's loop notices and bench_finish never keeps a root with no
170+
# index in it.
171+
if [ "$INDEX_RC" -ne 0 ] || [ -z "$PROJECT" ]; then
172+
echo " $LANG: index failed (cli exit $INDEX_RC, project '$PROJECT'); response in $OUT/00-index.json" >&2
173+
if [ -s "$INDEX_ERR" ]; then
174+
echo "--- index stderr ---" >&2
175+
cat "$INDEX_ERR" >&2
176+
fi
177+
echo "--- index response (first 500 bytes) ---" >&2
178+
printf '%.500s\n' "$INDEX_JSON" >&2
179+
exit 1
180+
fi
181+
81182
printf " %s: %s files, %s LOC, %sms, %s nodes, %s edges\n" \
82183
"$LANG" "$FILE_COUNT" "$LOC" "$ELAPSED" "$NODES" "$EDGES"

‎scripts/benchmark-search-graph.sh‎

Lines changed: 59 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,72 @@
33
# codebase-memory-mcp binary to measure the regex / LIKE pre-filter performance.
44
#
55
# Usage:
6-
# scripts/benchmark-search-graph.sh <binary-path> <project-name>
6+
# scripts/benchmark-search-graph.sh <binary-path> <repo-path>
77
#
88
# Example:
9-
# scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp my-project
9+
# scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp ~/src/my-project
10+
#
11+
# The repository is indexed (untimed) into a private runtime and cache first;
12+
# the queries then run against that index through a daemon this run keeps warm,
13+
# so a timing never includes daemon activation and never touches the operator's
14+
# live store (#1696).
1015

1116
set -euo pipefail
1217

13-
BINARY="${1:?Usage: $0 <binary-path> <project-name>}"
14-
PROJECT="${2:?Usage: $0 <binary-path> <project-name>}"
18+
BINARY="${1:?Usage: $0 <binary-path> <repo-path>}"
19+
REPO="${2:?Usage: $0 <binary-path> <repo-path>}"
20+
REPO=$(cd "$REPO" && pwd -P)
21+
22+
# shellcheck source=test-runtime.sh
23+
source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"
24+
cbm_test_runtime_init
25+
BENCH_TMP=""
26+
trap 'cbm_test_runtime_cleanup "$BINARY"; [ -z "$BENCH_TMP" ] || rm -rf -- "$BENCH_TMP"' EXIT
27+
BENCH_TMP=$(mktemp -d)
28+
INDEX_ERR="$BENCH_TMP/index-stderr.log"
29+
DAEMON_START_LOG="$BENCH_TMP/daemon-start.log"
30+
31+
# Its output is kept for the failure path: the product names why a daemon did
32+
# not start, and that is the part an operator needs.
33+
if ! "$BINARY" daemon start >"$DAEMON_START_LOG" 2>&1; then
34+
echo "private daemon did not start" >&2
35+
if [ -s "$DAEMON_START_LOG" ]; then
36+
echo "--- daemon start output ---" >&2
37+
cat "$DAEMON_START_LOG" >&2
38+
fi
39+
exit 1
40+
fi
41+
# One pre-escaped spelling of the path, the way the soak harness builds its
42+
# own: a repository path may legitimately contain a quote or a backslash, and
43+
# hand-built JSON turns that into a parse error.
44+
REPO_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$REPO")
45+
# Index and parse keep their stderr instead of discarding it: without it the
46+
# failure below names only its symptom, and the cause — an unreadable
47+
# repository, a refused daemon, a malformed envelope — is unrecoverable.
48+
INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" \
49+
2>"$INDEX_ERR" || echo '{}')
50+
PROJECT=$(echo "$INDEX_JSON" | python3 -c "
51+
import json, sys
52+
d = json.load(sys.stdin)
53+
if 'content' in d:
54+
d = json.loads(d['content'][0]['text'])
55+
print(d.get('project', ''))
56+
" 2>>"$INDEX_ERR" || echo "")
57+
if [ -z "$PROJECT" ]; then
58+
echo "index of $REPO did not report a project" >&2
59+
if [ -s "$INDEX_ERR" ]; then
60+
echo "--- index/parse stderr ---" >&2
61+
cat "$INDEX_ERR" >&2
62+
fi
63+
if [ -n "$INDEX_JSON" ]; then
64+
echo "--- index response (first 500 bytes) ---" >&2
65+
printf '%.500s\n' "$INDEX_JSON" >&2
66+
fi
67+
exit 1
68+
fi
1569

1670
echo "Binary: $BINARY"
17-
echo "Project: $PROJECT"
71+
echo "Project: $PROJECT (indexed from $REPO)"
1872
echo ""
1973

2074
run_case() {

‎scripts/test.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,9 @@ bash "$ROOT/tests/test_parallel_harness_contract.sh"
316316
echo "=== Step 0i2: suite failure-report contract ==="
317317
bash "$ROOT/tests/test_failure_report_contract.sh"
318318

319+
echo "=== Step 0i3: benchmark harness runtime isolation contract (#1696) ==="
320+
bash "$ROOT/tests/test_benchmark_runtime_isolation_contract.sh"
321+
319322
echo "=== Step 0j: venue parity contract (one harness, every venue) ==="
320323
bash "$ROOT/tests/test_venue_parity_contract.sh"
321324

0 commit comments

Comments
 (0)