Skip to content

Commit c9cacdf

Browse files
committed
perf(mcp): fast query tools on every platform
search_code - Scan the indexed files in process instead of through grep (POSIX) or a PowerShell Select-String pipeline (Windows), with the same grep -f semantics: one pattern per line, ERE regex, binary-file and NUL handling, -Hn records. The subprocess runner stays for the unindexed recursive fallback and a Windows file_pattern it cannot prefilter. - Windows now matches like macOS and Linux: Select-String matched case-insensitively, used .NET regex syntax and read NUL-containing files as text. A trailing CR is still ignored there, so `foo$` keeps matching CRLF checkouts. - Result merge, root containment and source reads no longer scale with hits x results: a node-id index, one canonical root per request with a per-file verdict cache, and fread spans instead of per-byte fgetc. query_graph - ORDER BY (RETURN and WITH) was a bubble sort. A stable merge sort keeps the same comparisons and so the same order; WITH keys are resolved once per row instead of twice per comparison. - GROUP BY, DISTINCT (RETURN and WITH) and COUNT(DISTINCT) looked every row up in a linear list. They hash the same keys now. Store reads - Read tools run their statements in one deferred read transaction, renewed every 100 ms, instead of taking the database file lock once per statement. list_projects - Resolve only the branch the listing prints: 2 git processes per project instead of 7. On macOS and Linux every response is byte-identical to before: search_code 30/30 (django), 28/28 (this repo) and 28/28 (Linux, GNU grep); all tools and the Cypher cases 86/86 on shared django and redis indexes. On django: search_code 908 -> 355 ms, WITH ... ORDER BY 41 s -> 62 ms, count(*) 508 -> 184 ms, list_projects 94 -> 40 ms. On the Windows ARM VM: search_code 1.6-3.4 s -> 0.2-0.5 s. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 0ca6e7d commit c9cacdf

11 files changed

Lines changed: 1860 additions & 140 deletions

File tree

‎src/cypher/cypher.c‎

Lines changed: 427 additions & 81 deletions
Large diffs are not rendered by default.

‎src/git/git_context.c‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,30 @@ int cbm_git_context_resolve(const char *path, cbm_git_context_t *out) {
313313
return 0;
314314
}
315315

316+
int cbm_git_branch_resolve(const char *path, bool *is_git, char **branch) {
317+
if (!is_git || !branch) {
318+
return CBM_NOT_FOUND;
319+
}
320+
*is_git = false;
321+
*branch = NULL;
322+
if (!path || !path[0]) {
323+
return CBM_NOT_FOUND;
324+
}
325+
if (!cbm_is_dir(path)) {
326+
return 0;
327+
}
328+
char *worktree_root = NULL;
329+
if (git_capture(path, "rev-parse --show-toplevel", &worktree_root) != 0) {
330+
return 0;
331+
}
332+
free(worktree_root);
333+
*is_git = true;
334+
if (git_capture(path, "symbolic-ref --quiet --short HEAD", branch) != 0) {
335+
*branch = git_strdup("DETACHED");
336+
}
337+
return *branch ? 0 : CBM_NOT_FOUND;
338+
}
339+
316340
char *cbm_git_context_branch_qn(const char *project_name, const cbm_git_context_t *ctx) {
317341
const char *project = project_name && project_name[0] ? project_name : "project";
318342
const char *slug = "working-tree";

‎src/git/git_context.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ typedef struct {
2020
} cbm_git_context_t;
2121

2222
int cbm_git_context_resolve(const char *path, cbm_git_context_t *out);
23+
/* Only the is_git and branch fields of cbm_git_context_resolve, from the same
24+
* two git commands it uses for them (2 git processes instead of 7). *branch
25+
* is heap-owned, NULL when path is not in a git work tree. */
26+
int cbm_git_branch_resolve(const char *path, bool *is_git, char **branch);
2327
void cbm_git_context_free(cbm_git_context_t *ctx);
2428
char *cbm_git_context_branch_qn(const char *project_name, const cbm_git_context_t *ctx);
2529
int cbm_git_context_props_json(const cbm_git_context_t *ctx, char *buf, int buf_size);

‎src/mcp/mcp.c‎

Lines changed: 674 additions & 59 deletions
Large diffs are not rendered by default.

‎src/mcp/mcp_internal.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,13 @@ void cbm_mcp_server_set_search_scan_timeout_for_test(cbm_mcp_server_t *srv, uint
3939
* scratch count cannot see other processes' entries in the shared temp dir.
4040
* The server borrows dir until it is reset or the server is freed. */
4141
void cbm_mcp_server_set_search_scratch_dir_for_test(cbm_mcp_server_t *srv, const char *dir);
42+
/* search_code's in-process scan of `filelist_path` (records separated as the
43+
* scoped list is written: NUL on POSIX, newline on Windows) into `out_path`.
44+
* Returns the scan cause in mcp_scan_cause_t order: 0 success, 1 command
45+
* failure, 3 output limit, 4 deadline, 5 cancelled. */
46+
int cbm_mcp_search_scan_for_test(cbm_mcp_server_t *srv, const char *filelist_path,
47+
const char *pattern, bool use_regex, const char *out_path,
48+
size_t output_limit, uint64_t deadline_ms);
4249

4350
/* Release only the constructor-created pristine in-memory store. Public
4451
* cbm_mcp_server_new(NULL) semantics remain unchanged; daemon sessions use

‎src/store/store.c‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,8 @@ struct cbm_store {
121121
const void *progress_owner; /* exclusive native progress guard token */
122122
const char *db_path; /* heap-allocated, or NULL for :memory: */
123123
char errbuf[CBM_SZ_512];
124+
bool read_batch; /* a read batch's transaction is open */
125+
uint64_t read_batch_started_ms; /* when it was last (re)begun */
124126

125127
/* Prepared statements (lazily initialized, cached for lifetime) */
126128
sqlite3_stmt *stmt_upsert_node;
@@ -207,10 +209,60 @@ static char *heap_strdup(const char *s) {
207209
}
208210

209211
/* Prepare a statement (cached). If already prepared, reset+clear. */
212+
/* ── Read batch ────────────────────────────────────────────────────
213+
* A read tool issues thousands of small statements. In autocommit mode each
214+
* one takes and drops the database file lock and stats the journal and WAL
215+
* files — most of a whole-graph count(*)'s time. A read batch runs them in one
216+
* deferred read transaction, renewed every ST_READ_BATCH_MS at the next
217+
* statement: a writer never waits behind the batch much longer than that, and
218+
* every statement still reads data committed at most that long ago (while the
219+
* batch holds a rollback-journal database's shared lock, no commit happens).
220+
* Committing a read transaction is allowed with read statements still
221+
* stepping; their lock is then held until they finish, as before. */
222+
enum { ST_READ_BATCH_MS = 100 };
223+
224+
bool cbm_store_read_batch_begin(cbm_store_t *s) {
225+
if (!s || !s->db || s->read_batch || !sqlite3_get_autocommit(s->db)) {
226+
return false;
227+
}
228+
if (sqlite3_exec(s->db, "BEGIN DEFERRED;", NULL, NULL, NULL) != SQLITE_OK) {
229+
return false;
230+
}
231+
s->read_batch = true;
232+
s->read_batch_started_ms = cbm_now_ms();
233+
return true;
234+
}
235+
236+
void cbm_store_read_batch_end(cbm_store_t *s) {
237+
if (!s || !s->read_batch) {
238+
return;
239+
}
240+
s->read_batch = false;
241+
if (s->db && !sqlite3_get_autocommit(s->db)) {
242+
(void)sqlite3_exec(s->db, "COMMIT;", NULL, NULL, NULL);
243+
}
244+
}
245+
246+
void cbm_store_read_batch_expire_for_test(cbm_store_t *s) {
247+
if (s && s->read_batch) {
248+
uint64_t now = cbm_now_ms();
249+
s->read_batch_started_ms = now > ST_READ_BATCH_MS ? now - ST_READ_BATCH_MS : 0;
250+
}
251+
}
252+
253+
static void read_batch_renew(cbm_store_t *s) {
254+
if (!s->read_batch || cbm_now_ms() - s->read_batch_started_ms < ST_READ_BATCH_MS) {
255+
return;
256+
}
257+
cbm_store_read_batch_end(s);
258+
(void)cbm_store_read_batch_begin(s);
259+
}
260+
210261
static sqlite3_stmt *prepare_cached(cbm_store_t *s, sqlite3_stmt **slot, const char *sql) {
211262
if (!s || !s->db) {
212263
return NULL;
213264
}
265+
read_batch_renew(s);
214266
if (*slot) {
215267
sqlite3_reset(*slot);
216268
sqlite3_clear_bindings(*slot);
@@ -1363,6 +1415,7 @@ void cbm_store_close(cbm_store_t *s) {
13631415
if (!s) {
13641416
return;
13651417
}
1418+
cbm_store_read_batch_end(s);
13661419

13671420
/* Checkpoint WAL before close to prevent orphan WAL accumulation.
13681421
* Best-effort — silently skips if concurrent reader holds a lock. */

‎src/store/store.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -478,6 +478,16 @@ int cbm_store_commit(cbm_store_t *s);
478478
/* Rollback the current transaction. */
479479
int cbm_store_rollback(cbm_store_t *s);
480480

481+
/* Read batch: run the reads that follow in one deferred read transaction,
482+
* renewed every 100 ms so a writer is never held back longer, instead of one
483+
* lock round trip per statement. Begin returns false (and does nothing) when
484+
* a transaction is already open. For read-only work only; end it before any
485+
* write, transaction or read scope on the same store. Close ends it. */
486+
bool cbm_store_read_batch_begin(cbm_store_t *s);
487+
void cbm_store_read_batch_end(cbm_store_t *s);
488+
/* Test seam: make an open read batch due for renewal at its next statement. */
489+
void cbm_store_read_batch_expire_for_test(cbm_store_t *s);
490+
481491
/* Stable request read scope. Borrows an exclusively owned, single-thread
482492
* store and callback context. Refuses a transaction, busy statement or owned
483493
* progress handler; idle cached statements are allowed. Saves query_only and

0 commit comments

Comments
 (0)