@@ -9,11 +9,74 @@ LANG="${2:?}"
99REPO=" ${3:? } "
1010RESULTS_DIR=" ${4:? } "
1111
12+ OUT=" $RESULTS_DIR /$LANG "
13+ mkdir -p " $OUT "
14+ # A handoff names this run's index or none: the evaluation loop reuses the
15+ # results directory, and a stale one would point the graph session at a root
16+ # step 8 has already removed. It goes before any setup step that can fail, so
17+ # a run that never reaches the index cannot leave an earlier one behind.
18+ rm -f -- " $OUT /runtime-root.txt"
19+
20+ # The index must run against a daemon rendezvous and cache this run owns: only
21+ # CBM_RUNTIME_DIR moves the rendezvous, and without a private cache the
22+ # benchmark repository was indexed into the operator's live store (#1696).
23+ # shellcheck source=test-runtime.sh
24+ source " $( dirname " ${BASH_SOURCE[0]} " ) /test-runtime.sh"
25+ cbm_test_runtime_init
26+
27+ # The evaluation plan (docs/EVALUATION_PLAN.md §7) indexes a language here and
28+ # then answers graph questions against that index from its own MCP session.
29+ # A run-private root would be gone before that session starts, so the caller
30+ # may ask for it to be kept: after a SUCCESSFUL run the harness stops its
31+ # daemon, leaves the root in place, records the paths that reach it in
32+ # <results>/<lang>/runtime-root.txt (sourceable), and ownership of the root —
33+ # including its removal — passes to the caller. A failed run cleans up
34+ # regardless: there is no index worth keeping, and nothing must leak. Never on
35+ # by default, or an unattended run accumulates one root per language.
36+ #
37+ # The handoff promises a stopped daemon, so it is written only once
38+ # `daemon stop` succeeded. A refused stop (a client still committed to the
39+ # daemon, or no answer) fails the run with the product's explanation, writes
40+ # no handoff, and keeps the root without cleaning it up: the daemon may still
41+ # be using it, and the operator decides what happens to it.
42+ bench_finish () {
43+ local rc=$? stop_out=" "
44+ [ -z " ${BENCH_TMP:- } " ] || rm -rf -- " $BENCH_TMP " || true
45+ if [ " $rc " -eq 0 ] && [ -n " ${CBM_BENCH_KEEP_RUNTIME:- } " ] && [ -d " ${OUT:- } " ]; then
46+ if ! stop_out=$( " $BINARY " daemon stop 2>&1 ) ; then
47+ echo " $LANG : private daemon did not stop; runtime kept at $CBM_TEST_RUNTIME_ROOT , no handoff written" >&2
48+ [ -z " $stop_out " ] || printf ' %s\n' " $stop_out " >&2
49+ exit 1
50+ fi
51+ printf ' CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \
52+ " $CBM_TEST_RUNTIME_ROOT " " $CBM_RUNTIME_DIR " " $CBM_CACHE_DIR " > " $OUT /runtime-root.txt"
53+ echo " $LANG : runtime kept at $CBM_TEST_RUNTIME_ROOT ; paths in $OUT /runtime-root.txt" >&2
54+ return 0
55+ fi
56+ cbm_test_runtime_cleanup " $BINARY "
57+ }
58+ BENCH_TMP=" "
59+ trap bench_finish EXIT
60+ # The index call keeps its stderr, the way the search-graph twin does: for a
61+ # one-shot CLI that is the only channel a refusal is reported on, and without
62+ # it the failure below could name only its symptom.
63+ BENCH_TMP=$( mktemp -d)
64+ INDEX_ERR=" $BENCH_TMP /index-stderr.log"
65+ DAEMON_START_LOG=" $BENCH_TMP /daemon-start.log"
66+
1267# Resolve symlinks
1368REPO=$( cd " $REPO " && pwd -P)
69+ # One pre-escaped spelling of the path for every request below, the way the
70+ # soak harness builds its own: a repository path may legitimately contain a
71+ # quote or a backslash, and hand-built JSON turns that into a parse error.
72+ REPO_JSON=$( python3 -c ' import json,sys; print(json.dumps(sys.argv[1]))' " $REPO " )
1473
15- OUT=" $RESULTS_DIR /$LANG "
16- mkdir -p " $OUT "
74+ # Elapsed time is read from a monotonic clock, never the wall clock: an NTP
75+ # step mid-run would otherwise skew — or negate — a figure whose whole purpose
76+ # is comparison across runs. Its reference point is fixed per boot on every
77+ # platform CPython supports here, so the three readings below are comparable
78+ # even though each comes from its own process.
79+ bench_now_ms () { python3 -c " import time; print(time.monotonic_ns() // 1000000)" ; }
1780
1881echo " INDEX: $LANG ($REPO )"
1982
@@ -33,16 +96,39 @@ LOC=$(find "$REPO" -type f \
3396echo " $FILE_COUNT " > " $OUT /file-count.txt"
3497echo " $LOC " > " $OUT /loc.txt"
3598
99+ # Start the private daemon before timing so index-time.txt measures the index
100+ # alone. setup-time.txt keeps the activation cost attributable and
101+ # total-time.txt is their sum — the figure comparable with earlier runs, which
102+ # paid activation inside the index timing whenever no daemon was already warm.
103+ SETUP_START_MS=$( bench_now_ms)
104+ # Its output is kept for the failure path: the product names why a daemon did
105+ # not start, and that is the part an operator needs.
106+ if ! " $BINARY " daemon start > " $DAEMON_START_LOG " 2>&1 ; then
107+ echo " $LANG : private daemon did not start" >&2
108+ if [ -s " $DAEMON_START_LOG " ]; then
109+ echo " --- daemon start output ---" >&2
110+ cat " $DAEMON_START_LOG " >&2
111+ fi
112+ exit 1
113+ fi
114+
36115# Index via CLI and capture timing
37- START_MS=$( python3 -c " import time; print(int(time.time()*1000)) " )
116+ START_MS=$( bench_now_ms )
38117
39- INDEX_JSON=$( " $BINARY " cli index_repository " {\" repo_path\" :\" $REPO \" ,\" mode\" :\" full\" }" 2> /dev/null || echo ' {"error":"index failed"}' )
118+ # The CLI's exit status is the index's verdict. It is recorded here and acted
119+ # on below, once every per-run file is written; an empty response stays valid
120+ # JSON in 00-index.json so the failure is legible there as well.
121+ INDEX_RC=0
122+ INDEX_JSON=$( " $BINARY " cli index_repository " {\" repo_path\" :$REPO_JSON ,\" mode\" :\" full\" }" 2> " $INDEX_ERR " ) ||
123+ { INDEX_RC=$? ; INDEX_JSON=" {\" error\" :\" index failed\" ,\" exit\" :$INDEX_RC }" ; }
40124
41- END_MS=$( python3 -c " import time; print(int(time.time()*1000)) " )
125+ END_MS=$( bench_now_ms )
42126ELAPSED=$(( END_MS - START_MS))
43127
44128echo " $INDEX_JSON " > " $OUT /00-index.json"
45129echo " $ELAPSED " > " $OUT /index-time.txt"
130+ echo " $(( START_MS - SETUP_START_MS)) " > " $OUT /setup-time.txt"
131+ echo " $(( END_MS - SETUP_START_MS)) " > " $OUT /total-time.txt"
46132
47133# Extract node/edge counts (CLI wraps in MCP content envelope)
48134NODES=$( echo " $INDEX_JSON " | python3 -c "
@@ -78,5 +164,20 @@ echo "$NODES" > "$OUT/nodes.txt"
78164echo " $EDGES " > " $OUT /edges.txt"
79165echo " $PROJECT " > " $OUT /project.txt"
80166
167+ # An index that failed, or that named no project, fails the run: after the
168+ # timing files, so the caller keeps its figures, and through the exit status,
169+ # so the caller's loop notices and bench_finish never keeps a root with no
170+ # index in it.
171+ if [ " $INDEX_RC " -ne 0 ] || [ -z " $PROJECT " ]; then
172+ echo " $LANG : index failed (cli exit $INDEX_RC , project '$PROJECT '); response in $OUT /00-index.json" >&2
173+ if [ -s " $INDEX_ERR " ]; then
174+ echo " --- index stderr ---" >&2
175+ cat " $INDEX_ERR " >&2
176+ fi
177+ echo " --- index response (first 500 bytes) ---" >&2
178+ printf ' %.500s\n' " $INDEX_JSON " >&2
179+ exit 1
180+ fi
181+
81182printf " %s: %s files, %s LOC, %sms, %s nodes, %s edges\n" \
82183 " $LANG " " $FILE_COUNT " " $LOC " " $ELAPSED " " $NODES " " $EDGES "
0 commit comments