Skip to content

Commit b95b403

Browse files
committed
fix(mcp): rank every BM25 candidate when post-window filters would hide them
file_pattern, kind and namespace are applied after the 2000-row BM25 candidate window: unlike the label they cannot go into the FTS5 MATCH. With a common query token and a narrow filter the window held none of the filtered rows, and search_graph answered 0 rows -- or a few rows of an under-counted total -- although matching nodes exist. An unlabelled query whose window held only File/Folder/Variable/Project rows had the same failure. When the window is saturated and such a filter is set, or the window held no row passing the filters, the query is now ranked over every candidate (inner LIMIT -1): the FTS-first subquery plan stays, the extra work is linear in the token's match count (the saturation probe already orders every match), and the total is exact. The decision depends only on the query, never on the page, so offset pages stay consistent. Unfiltered and label-only queries that find rows keep the window and are byte-identical. The filter binds, the count query and the saturation probe move into helpers shared by the ranked and the count query. Measured on Django, meilisearch and kubernetes indexes (9 / 9 / 24 filtered queries): zero results 6 -> 0, 6 -> 0, 23 -> 0; rows returned 52 -> 156, 3 -> 303, 19 -> 807; exact totals 0 -> 9, 0 -> 9, 0 -> 24. The graph is unchanged (search-only change). Refs DeusData#2386 Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 8af6833 commit b95b403

2 files changed

Lines changed: 273 additions & 107 deletions

File tree

‎src/mcp/mcp.c‎

Lines changed: 121 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -4019,6 +4019,8 @@ enum {
40194019
* and then joining/filtering/re-ranking those, we bound all work to O(N) where
40204020
* N = BM25_INNER_LIMIT rather than the full match set size. */
40214021
BM25_INNER_LIMIT = 2000,
4022+
/* SQLite reads a negative LIMIT as "no limit": rank every candidate. */
4023+
BM25_INNER_ALL = -1,
40224024
/* Longest label spliced into the MATCH pre-filter; real labels are short
40234025
* identifiers, so anything longer simply skips the pre-filter. */
40244026
BM25_LABEL_MAX = 64,
@@ -4374,6 +4376,96 @@ static char *bm25_render(const bm25_output_row_t *rows, int returned, int total,
43744376
return json;
43754377
}
43764378

4379+
/* The MATCH expression and the filters the ranked and the count query share.
4380+
* One binder for both keeps them on identical parameters. */
4381+
typedef struct {
4382+
const char *fts_query;
4383+
const char *project;
4384+
const char *file_like;
4385+
const char *label;
4386+
const char *kind;
4387+
const char *ns;
4388+
double label_weight;
4389+
} bm25_filters_t;
4390+
4391+
static void bm25_bind_opt_text(sqlite3_stmt *st, int idx, const char *value) {
4392+
if (value && value[0]) {
4393+
sqlite3_bind_text(st, idx, value, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4394+
} else {
4395+
sqlite3_bind_null(st, idx);
4396+
}
4397+
}
4398+
4399+
static void bm25_bind_filters(sqlite3_stmt *st, const bm25_filters_t *f) {
4400+
sqlite3_bind_text(st, BM25_BIND_QUERY, f->fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4401+
sqlite3_bind_text(st, BM25_BIND_PROJECT, f->project, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4402+
bm25_bind_opt_text(st, BM25_BIND_FILE, f->file_like);
4403+
bm25_bind_opt_text(st, BM25_BIND_LABEL, f->label);
4404+
bm25_bind_opt_text(st, BM25_BIND_KIND, f->kind);
4405+
bm25_bind_opt_text(st, BM25_BIND_NS, f->ns);
4406+
sqlite3_bind_double(st, BM25_BIND_LABEL_WEIGHT, f->label_weight);
4407+
}
4408+
4409+
/* The top-candidate window is a performance ceiling, not an exact-total
4410+
* boundary. Probe one candidate beyond it so a broad query never presents a
4411+
* window-local count as the complete match count. The probe shares the
4412+
* label-scoped MATCH but is otherwise global to the FTS table; saturation is
4413+
* therefore conservatively reported even when later project/path filters
4414+
* might discard the hidden candidates. */
4415+
static bool bm25_window_saturated(sqlite3 *db, const char *fts_query) {
4416+
enum { PROBE_BIND_OFFSET = 2 };
4417+
const char *probe_sql = "SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1 "
4418+
"ORDER BY bm25(nodes_fts), rowid LIMIT 1 OFFSET ?2";
4419+
sqlite3_stmt *probe = NULL;
4420+
if (sqlite3_prepare_v2(db, probe_sql, BM25_SQL_AUTO_LEN, &probe, NULL) != SQLITE_OK) {
4421+
return true;
4422+
}
4423+
sqlite3_bind_text(probe, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4424+
sqlite3_bind_int(probe, PROBE_BIND_OFFSET, BM25_INNER_LIMIT);
4425+
bool saturated = sqlite3_step(probe) != SQLITE_DONE;
4426+
sqlite3_finalize(probe);
4427+
return saturated;
4428+
}
4429+
4430+
/* Count the candidates that pass the filters among the top `inner_limit`
4431+
* (BM25_INNER_ALL: among all of them), with the ranked query's subquery shape
4432+
* so FTS5 drives the scan here too. */
4433+
static int bm25_count_candidates(sqlite3 *db, const bm25_filters_t *f, int inner_limit) {
4434+
const char *count_sql =
4435+
"SELECT COUNT(*) FROM ("
4436+
" SELECT fts.rowid FROM ("
4437+
" SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1"
4438+
" ORDER BY " BM25_WEIGHTS " LIMIT ?3"
4439+
" ) fts "
4440+
" JOIN nodes n ON n.id = fts.rowid "
4441+
" WHERE n.project = ?2 "
4442+
/* MIRRORS the ranked query's filter verbatim — same weights, same
4443+
* label exclusions. Changing one alone reports a total that does
4444+
* not describe the rows returned. */
4445+
" AND " BM25_UNLABELLED_EXCLUSION " "
4446+
" AND (?6 IS NULL OR n.file_path LIKE ?6)"
4447+
" AND (?7 IS NULL OR n.label = ?7)"
4448+
" AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) "
4449+
" THEN json_extract(n.properties,'$.kind') END) = ?9)"
4450+
" AND (?10 IS NULL OR (CASE WHEN json_valid(n.properties) "
4451+
" THEN json_extract(n.properties,'$.obj_key') END) "
4452+
" LIKE ('%/%/' || replace(replace(replace(?10,'\\','\\\\'),'%','\\%'),"
4453+
" '_','\\_') || '/%') ESCAPE '\\')"
4454+
")";
4455+
int total = 0;
4456+
sqlite3_stmt *cs = NULL;
4457+
if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) != SQLITE_OK) {
4458+
return 0;
4459+
}
4460+
bm25_bind_filters(cs, f);
4461+
sqlite3_bind_int(cs, BM25_BIND_LIMIT, inner_limit);
4462+
if (sqlite3_step(cs) == SQLITE_ROW) {
4463+
total = sqlite3_column_int(cs, 0);
4464+
}
4465+
sqlite3_finalize(cs);
4466+
return total;
4467+
}
4468+
43774469
/* Run the BM25 full-text search path and return the JSON result string.
43784470
* Returns NULL if FTS5 is unavailable or the query produced no usable tokens,
43794471
* in which case the caller falls back to the regex-based search path. */
@@ -4461,117 +4553,39 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
44614553
free(file_like);
44624554
return NULL;
44634555
}
4464-
sqlite3_bind_text(stmt, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4465-
sqlite3_bind_text(stmt, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4556+
const bm25_filters_t filters = {.fts_query = fts_query,
4557+
.project = project,
4558+
.file_like = file_like,
4559+
.label = label,
4560+
.kind = kind,
4561+
.ns = ns,
4562+
.label_weight = label_weight};
4563+
bm25_bind_filters(stmt, &filters);
44664564
sqlite3_bind_int(stmt, BM25_BIND_LIMIT, limit > 0 ? limit : BM25_DEFAULT_LIMIT);
44674565
sqlite3_bind_int(stmt, BM25_BIND_OFFSET, offset > 0 ? offset : 0);
4468-
sqlite3_bind_int(stmt, BM25_BIND_INNER, BM25_INNER_LIMIT);
4469-
if (file_like) {
4470-
sqlite3_bind_text(stmt, BM25_BIND_FILE, file_like, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4471-
} else {
4472-
sqlite3_bind_null(stmt, BM25_BIND_FILE);
4473-
}
4474-
if (label && label[0]) {
4475-
sqlite3_bind_text(stmt, BM25_BIND_LABEL, label, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4476-
} else {
4477-
sqlite3_bind_null(stmt, BM25_BIND_LABEL);
4478-
}
44794566
sqlite3_bind_text(stmt, BM25_BIND_EXACT, query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4480-
if (kind && kind[0]) {
4481-
sqlite3_bind_text(stmt, BM25_BIND_KIND, kind, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4482-
} else {
4483-
sqlite3_bind_null(stmt, BM25_BIND_KIND);
4484-
}
4485-
if (ns && ns[0]) {
4486-
sqlite3_bind_text(stmt, BM25_BIND_NS, ns, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4487-
} else {
4488-
sqlite3_bind_null(stmt, BM25_BIND_NS);
4489-
}
4490-
sqlite3_bind_double(stmt, BM25_BIND_LABEL_WEIGHT, label_weight);
4491-
4492-
/* Count hits within the same inner-limit window — capped at BM25_INNER_LIMIT.
4493-
* Uses the identical subquery structure so the FTS5 early-exit applies here too. */
4494-
int total = 0;
4495-
{
4496-
const char *count_sql =
4497-
"SELECT COUNT(*) FROM ("
4498-
" SELECT fts.rowid FROM ("
4499-
" SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1"
4500-
" ORDER BY " BM25_WEIGHTS " LIMIT ?3"
4501-
" ) fts "
4502-
" JOIN nodes n ON n.id = fts.rowid "
4503-
" WHERE n.project = ?2 "
4504-
/* MIRRORS the ranked query's filter verbatim — same weights, same
4505-
* label exclusions. Changing one alone reports a total that does
4506-
* not describe the rows returned. */
4507-
" AND " BM25_UNLABELLED_EXCLUSION " "
4508-
" AND (?6 IS NULL OR n.file_path LIKE ?6)"
4509-
" AND (?7 IS NULL OR n.label = ?7)"
4510-
" AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) "
4511-
" THEN json_extract(n.properties,'$.kind') END) = ?9)"
4512-
" AND (?10 IS NULL OR (CASE WHEN json_valid(n.properties) "
4513-
" THEN json_extract(n.properties,'$.obj_key') END) "
4514-
" LIKE ('%/%/' || replace(replace(replace(?10,'\\','\\\\'),'%','\\%'),"
4515-
" '_','\\_') || '/%') ESCAPE '\\')"
4516-
")";
4517-
sqlite3_stmt *cs = NULL;
4518-
if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) == SQLITE_OK) {
4519-
sqlite3_bind_text(cs, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN,
4520-
MCP_SQLITE_TRANSIENT);
4521-
sqlite3_bind_text(cs, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN,
4522-
MCP_SQLITE_TRANSIENT);
4523-
sqlite3_bind_int(cs, BM25_BIND_LIMIT, BM25_INNER_LIMIT);
4524-
sqlite3_bind_double(cs, BM25_BIND_LABEL_WEIGHT, label_weight);
4525-
if (file_like) {
4526-
sqlite3_bind_text(cs, BM25_BIND_FILE, file_like, BM25_SQL_AUTO_LEN,
4527-
MCP_SQLITE_TRANSIENT);
4528-
} else {
4529-
sqlite3_bind_null(cs, BM25_BIND_FILE);
4530-
}
4531-
if (label && label[0]) {
4532-
sqlite3_bind_text(cs, BM25_BIND_LABEL, label, BM25_SQL_AUTO_LEN,
4533-
MCP_SQLITE_TRANSIENT);
4534-
} else {
4535-
sqlite3_bind_null(cs, BM25_BIND_LABEL);
4536-
}
4537-
if (kind && kind[0]) {
4538-
sqlite3_bind_text(cs, BM25_BIND_KIND, kind, BM25_SQL_AUTO_LEN,
4539-
MCP_SQLITE_TRANSIENT);
4540-
} else {
4541-
sqlite3_bind_null(cs, BM25_BIND_KIND);
4542-
}
4543-
if (ns && ns[0]) {
4544-
sqlite3_bind_text(cs, BM25_BIND_NS, ns, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4545-
} else {
4546-
sqlite3_bind_null(cs, BM25_BIND_NS);
4547-
}
4548-
if (sqlite3_step(cs) == SQLITE_ROW) {
4549-
total = sqlite3_column_int(cs, 0);
4550-
}
4551-
sqlite3_finalize(cs);
4552-
}
4553-
}
45544567

4555-
/* The top-candidate window is a performance ceiling, not an exact-total
4556-
* boundary. Probe one candidate beyond it so a broad query never presents
4557-
* a window-local count as the complete match count. The probe shares the
4558-
* label-scoped MATCH but is otherwise global to the FTS table; saturation
4559-
* is therefore conservatively reported even when later project/path
4560-
* filters might discard the hidden candidates. */
4561-
bool candidate_window_saturated = true;
4562-
{
4563-
const char *probe_sql = "SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1 "
4564-
"ORDER BY bm25(nodes_fts), rowid LIMIT 1 OFFSET ?2";
4565-
sqlite3_stmt *probe = NULL;
4566-
if (sqlite3_prepare_v2(db, probe_sql, BM25_SQL_AUTO_LEN, &probe, NULL) == SQLITE_OK) {
4567-
sqlite3_bind_text(probe, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN,
4568-
MCP_SQLITE_TRANSIENT);
4569-
sqlite3_bind_int(probe, BM25_BIND_PROJECT, BM25_INNER_LIMIT);
4570-
int probe_step = sqlite3_step(probe);
4571-
candidate_window_saturated = probe_step != SQLITE_DONE;
4572-
sqlite3_finalize(probe);
4573-
}
4574-
}
4568+
/* file_pattern, kind and namespace cannot be put into the MATCH the way
4569+
* the label is, so they filter AFTER the candidate window. When more
4570+
* candidates exist beyond it, a narrow filter can find none of its rows in
4571+
* the top BM25_INNER_LIMIT and the search answered 0 rows although
4572+
* matching nodes exist. A saturated window under such a filter -- or one
4573+
* that held no row passing the filters at all -- is therefore ranked over
4574+
* every candidate instead: the FTS-first subquery plan stays, the extra
4575+
* work is linear in the token's match count (the saturation probe already
4576+
* orders every match), and the total becomes exact. The decision depends
4577+
* only on the query, never on the page, so offset pages stay consistent.
4578+
* Unfiltered and label-only queries that find rows keep the window. */
4579+
bool saturated = bm25_window_saturated(db, fts_query);
4580+
bool post_window_filter = file_like || (kind && kind[0]) || (ns && ns[0]);
4581+
int inner_limit = saturated && post_window_filter ? BM25_INNER_ALL : BM25_INNER_LIMIT;
4582+
int total = bm25_count_candidates(db, &filters, inner_limit);
4583+
if (saturated && total == 0 && inner_limit != BM25_INNER_ALL) {
4584+
inner_limit = BM25_INNER_ALL;
4585+
total = bm25_count_candidates(db, &filters, inner_limit);
4586+
}
4587+
bool candidate_window_saturated = saturated && inner_limit != BM25_INNER_ALL;
4588+
sqlite3_bind_int(stmt, BM25_BIND_INNER, inner_limit);
45754589

45764590
int row_cap = limit > 0 ? limit : BM25_DEFAULT_LIMIT;
45774591
bm25_output_row_t *rows =

0 commit comments

Comments
 (0)