Skip to content

Commit e082387

Browse files
authored
Merge pull request DeusData#2819 from DeusData/fix/2386-bm25-post-window-filters
fix(mcp): rank every BM25 candidate when post-window filters would hide them
2 parents d1d8bbf + d86ca26 commit e082387

2 files changed

Lines changed: 273 additions & 107 deletions

File tree

‎src/mcp/mcp.c‎

Lines changed: 121 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -4036,6 +4036,8 @@ enum {
40364036
* and then joining/filtering/re-ranking those, we bound all work to O(N) where
40374037
* N = BM25_INNER_LIMIT rather than the full match set size. */
40384038
BM25_INNER_LIMIT = 2000,
4039+
/* SQLite reads a negative LIMIT as "no limit": rank every candidate. */
4040+
BM25_INNER_ALL = -1,
40394041
/* Longest label spliced into the MATCH pre-filter; real labels are short
40404042
* identifiers, so anything longer simply skips the pre-filter. */
40414043
BM25_LABEL_MAX = 64,
@@ -4391,6 +4393,96 @@ static char *bm25_render(const bm25_output_row_t *rows, int returned, int total,
43914393
return json;
43924394
}
43934395

4396+
/* The MATCH expression and the filters the ranked and the count query share.
4397+
* One binder for both keeps them on identical parameters. */
4398+
typedef struct {
4399+
const char *fts_query;
4400+
const char *project;
4401+
const char *file_like;
4402+
const char *label;
4403+
const char *kind;
4404+
const char *ns;
4405+
double label_weight;
4406+
} bm25_filters_t;
4407+
4408+
static void bm25_bind_opt_text(sqlite3_stmt *st, int idx, const char *value) {
4409+
if (value && value[0]) {
4410+
sqlite3_bind_text(st, idx, value, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4411+
} else {
4412+
sqlite3_bind_null(st, idx);
4413+
}
4414+
}
4415+
4416+
static void bm25_bind_filters(sqlite3_stmt *st, const bm25_filters_t *f) {
4417+
sqlite3_bind_text(st, BM25_BIND_QUERY, f->fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4418+
sqlite3_bind_text(st, BM25_BIND_PROJECT, f->project, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4419+
bm25_bind_opt_text(st, BM25_BIND_FILE, f->file_like);
4420+
bm25_bind_opt_text(st, BM25_BIND_LABEL, f->label);
4421+
bm25_bind_opt_text(st, BM25_BIND_KIND, f->kind);
4422+
bm25_bind_opt_text(st, BM25_BIND_NS, f->ns);
4423+
sqlite3_bind_double(st, BM25_BIND_LABEL_WEIGHT, f->label_weight);
4424+
}
4425+
4426+
/* The top-candidate window is a performance ceiling, not an exact-total
4427+
* boundary. Probe one candidate beyond it so a broad query never presents a
4428+
* window-local count as the complete match count. The probe shares the
4429+
* label-scoped MATCH but is otherwise global to the FTS table; saturation is
4430+
* therefore conservatively reported even when later project/path filters
4431+
* might discard the hidden candidates. */
4432+
static bool bm25_window_saturated(sqlite3 *db, const char *fts_query) {
4433+
enum { PROBE_BIND_OFFSET = 2 };
4434+
const char *probe_sql = "SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1 "
4435+
"ORDER BY bm25(nodes_fts), rowid LIMIT 1 OFFSET ?2";
4436+
sqlite3_stmt *probe = NULL;
4437+
if (sqlite3_prepare_v2(db, probe_sql, BM25_SQL_AUTO_LEN, &probe, NULL) != SQLITE_OK) {
4438+
return true;
4439+
}
4440+
sqlite3_bind_text(probe, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4441+
sqlite3_bind_int(probe, PROBE_BIND_OFFSET, BM25_INNER_LIMIT);
4442+
bool saturated = sqlite3_step(probe) != SQLITE_DONE;
4443+
sqlite3_finalize(probe);
4444+
return saturated;
4445+
}
4446+
4447+
/* Count the candidates that pass the filters among the top `inner_limit`
4448+
* (BM25_INNER_ALL: among all of them), with the ranked query's subquery shape
4449+
* so FTS5 drives the scan here too. */
4450+
static int bm25_count_candidates(sqlite3 *db, const bm25_filters_t *f, int inner_limit) {
4451+
const char *count_sql =
4452+
"SELECT COUNT(*) FROM ("
4453+
" SELECT fts.rowid FROM ("
4454+
" SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1"
4455+
" ORDER BY " BM25_WEIGHTS " LIMIT ?3"
4456+
" ) fts "
4457+
" JOIN nodes n ON n.id = fts.rowid "
4458+
" WHERE n.project = ?2 "
4459+
/* MIRRORS the ranked query's filter verbatim — same weights, same
4460+
* label exclusions. Changing one alone reports a total that does
4461+
* not describe the rows returned. */
4462+
" AND " BM25_UNLABELLED_EXCLUSION " "
4463+
" AND (?6 IS NULL OR n.file_path LIKE ?6)"
4464+
" AND (?7 IS NULL OR n.label = ?7)"
4465+
" AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) "
4466+
" THEN json_extract(n.properties,'$.kind') END) = ?9)"
4467+
" AND (?10 IS NULL OR (CASE WHEN json_valid(n.properties) "
4468+
" THEN json_extract(n.properties,'$.obj_key') END) "
4469+
" LIKE ('%/%/' || replace(replace(replace(?10,'\\','\\\\'),'%','\\%'),"
4470+
" '_','\\_') || '/%') ESCAPE '\\')"
4471+
")";
4472+
int total = 0;
4473+
sqlite3_stmt *cs = NULL;
4474+
if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) != SQLITE_OK) {
4475+
return 0;
4476+
}
4477+
bm25_bind_filters(cs, f);
4478+
sqlite3_bind_int(cs, BM25_BIND_LIMIT, inner_limit);
4479+
if (sqlite3_step(cs) == SQLITE_ROW) {
4480+
total = sqlite3_column_int(cs, 0);
4481+
}
4482+
sqlite3_finalize(cs);
4483+
return total;
4484+
}
4485+
43944486
/* Run the BM25 full-text search path and return the JSON result string.
43954487
* Returns NULL if FTS5 is unavailable or the query produced no usable tokens,
43964488
* in which case the caller falls back to the regex-based search path. */
@@ -4478,117 +4570,39 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
44784570
free(file_like);
44794571
return NULL;
44804572
}
4481-
sqlite3_bind_text(stmt, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4482-
sqlite3_bind_text(stmt, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4573+
const bm25_filters_t filters = {.fts_query = fts_query,
4574+
.project = project,
4575+
.file_like = file_like,
4576+
.label = label,
4577+
.kind = kind,
4578+
.ns = ns,
4579+
.label_weight = label_weight};
4580+
bm25_bind_filters(stmt, &filters);
44834581
sqlite3_bind_int(stmt, BM25_BIND_LIMIT, limit > 0 ? limit : BM25_DEFAULT_LIMIT);
44844582
sqlite3_bind_int(stmt, BM25_BIND_OFFSET, offset > 0 ? offset : 0);
4485-
sqlite3_bind_int(stmt, BM25_BIND_INNER, BM25_INNER_LIMIT);
4486-
if (file_like) {
4487-
sqlite3_bind_text(stmt, BM25_BIND_FILE, file_like, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4488-
} else {
4489-
sqlite3_bind_null(stmt, BM25_BIND_FILE);
4490-
}
4491-
if (label && label[0]) {
4492-
sqlite3_bind_text(stmt, BM25_BIND_LABEL, label, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4493-
} else {
4494-
sqlite3_bind_null(stmt, BM25_BIND_LABEL);
4495-
}
44964583
sqlite3_bind_text(stmt, BM25_BIND_EXACT, query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4497-
if (kind && kind[0]) {
4498-
sqlite3_bind_text(stmt, BM25_BIND_KIND, kind, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4499-
} else {
4500-
sqlite3_bind_null(stmt, BM25_BIND_KIND);
4501-
}
4502-
if (ns && ns[0]) {
4503-
sqlite3_bind_text(stmt, BM25_BIND_NS, ns, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4504-
} else {
4505-
sqlite3_bind_null(stmt, BM25_BIND_NS);
4506-
}
4507-
sqlite3_bind_double(stmt, BM25_BIND_LABEL_WEIGHT, label_weight);
45084584

4509-
/* Count hits within the same inner-limit window — capped at BM25_INNER_LIMIT.
4510-
* Uses the identical subquery structure so the FTS5 early-exit applies here too. */
4511-
int total = 0;
4512-
{
4513-
const char *count_sql =
4514-
"SELECT COUNT(*) FROM ("
4515-
" SELECT fts.rowid FROM ("
4516-
" SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1"
4517-
" ORDER BY " BM25_WEIGHTS " LIMIT ?3"
4518-
" ) fts "
4519-
" JOIN nodes n ON n.id = fts.rowid "
4520-
" WHERE n.project = ?2 "
4521-
/* MIRRORS the ranked query's filter verbatim — same weights, same
4522-
* label exclusions. Changing one alone reports a total that does
4523-
* not describe the rows returned. */
4524-
" AND " BM25_UNLABELLED_EXCLUSION " "
4525-
" AND (?6 IS NULL OR n.file_path LIKE ?6)"
4526-
" AND (?7 IS NULL OR n.label = ?7)"
4527-
" AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) "
4528-
" THEN json_extract(n.properties,'$.kind') END) = ?9)"
4529-
" AND (?10 IS NULL OR (CASE WHEN json_valid(n.properties) "
4530-
" THEN json_extract(n.properties,'$.obj_key') END) "
4531-
" LIKE ('%/%/' || replace(replace(replace(?10,'\\','\\\\'),'%','\\%'),"
4532-
" '_','\\_') || '/%') ESCAPE '\\')"
4533-
")";
4534-
sqlite3_stmt *cs = NULL;
4535-
if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) == SQLITE_OK) {
4536-
sqlite3_bind_text(cs, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN,
4537-
MCP_SQLITE_TRANSIENT);
4538-
sqlite3_bind_text(cs, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN,
4539-
MCP_SQLITE_TRANSIENT);
4540-
sqlite3_bind_int(cs, BM25_BIND_LIMIT, BM25_INNER_LIMIT);
4541-
sqlite3_bind_double(cs, BM25_BIND_LABEL_WEIGHT, label_weight);
4542-
if (file_like) {
4543-
sqlite3_bind_text(cs, BM25_BIND_FILE, file_like, BM25_SQL_AUTO_LEN,
4544-
MCP_SQLITE_TRANSIENT);
4545-
} else {
4546-
sqlite3_bind_null(cs, BM25_BIND_FILE);
4547-
}
4548-
if (label && label[0]) {
4549-
sqlite3_bind_text(cs, BM25_BIND_LABEL, label, BM25_SQL_AUTO_LEN,
4550-
MCP_SQLITE_TRANSIENT);
4551-
} else {
4552-
sqlite3_bind_null(cs, BM25_BIND_LABEL);
4553-
}
4554-
if (kind && kind[0]) {
4555-
sqlite3_bind_text(cs, BM25_BIND_KIND, kind, BM25_SQL_AUTO_LEN,
4556-
MCP_SQLITE_TRANSIENT);
4557-
} else {
4558-
sqlite3_bind_null(cs, BM25_BIND_KIND);
4559-
}
4560-
if (ns && ns[0]) {
4561-
sqlite3_bind_text(cs, BM25_BIND_NS, ns, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4562-
} else {
4563-
sqlite3_bind_null(cs, BM25_BIND_NS);
4564-
}
4565-
if (sqlite3_step(cs) == SQLITE_ROW) {
4566-
total = sqlite3_column_int(cs, 0);
4567-
}
4568-
sqlite3_finalize(cs);
4569-
}
4570-
}
4571-
4572-
/* The top-candidate window is a performance ceiling, not an exact-total
4573-
* boundary. Probe one candidate beyond it so a broad query never presents
4574-
* a window-local count as the complete match count. The probe shares the
4575-
* label-scoped MATCH but is otherwise global to the FTS table; saturation
4576-
* is therefore conservatively reported even when later project/path
4577-
* filters might discard the hidden candidates. */
4578-
bool candidate_window_saturated = true;
4579-
{
4580-
const char *probe_sql = "SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1 "
4581-
"ORDER BY bm25(nodes_fts), rowid LIMIT 1 OFFSET ?2";
4582-
sqlite3_stmt *probe = NULL;
4583-
if (sqlite3_prepare_v2(db, probe_sql, BM25_SQL_AUTO_LEN, &probe, NULL) == SQLITE_OK) {
4584-
sqlite3_bind_text(probe, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN,
4585-
MCP_SQLITE_TRANSIENT);
4586-
sqlite3_bind_int(probe, BM25_BIND_PROJECT, BM25_INNER_LIMIT);
4587-
int probe_step = sqlite3_step(probe);
4588-
candidate_window_saturated = probe_step != SQLITE_DONE;
4589-
sqlite3_finalize(probe);
4590-
}
4591-
}
4585+
/* file_pattern, kind and namespace cannot be put into the MATCH the way
4586+
* the label is, so they filter AFTER the candidate window. When more
4587+
* candidates exist beyond it, a narrow filter can find none of its rows in
4588+
* the top BM25_INNER_LIMIT and the search answered 0 rows although
4589+
* matching nodes exist. A saturated window under such a filter -- or one
4590+
* that held no row passing the filters at all -- is therefore ranked over
4591+
* every candidate instead: the FTS-first subquery plan stays, the extra
4592+
* work is linear in the token's match count (the saturation probe already
4593+
* orders every match), and the total becomes exact. The decision depends
4594+
* only on the query, never on the page, so offset pages stay consistent.
4595+
* Unfiltered and label-only queries that find rows keep the window. */
4596+
bool saturated = bm25_window_saturated(db, fts_query);
4597+
bool post_window_filter = file_like || (kind && kind[0]) || (ns && ns[0]);
4598+
int inner_limit = saturated && post_window_filter ? BM25_INNER_ALL : BM25_INNER_LIMIT;
4599+
int total = bm25_count_candidates(db, &filters, inner_limit);
4600+
if (saturated && total == 0 && inner_limit != BM25_INNER_ALL) {
4601+
inner_limit = BM25_INNER_ALL;
4602+
total = bm25_count_candidates(db, &filters, inner_limit);
4603+
}
4604+
bool candidate_window_saturated = saturated && inner_limit != BM25_INNER_ALL;
4605+
sqlite3_bind_int(stmt, BM25_BIND_INNER, inner_limit);
45924606

45934607
int row_cap = limit > 0 ? limit : BM25_DEFAULT_LIMIT;
45944608
bm25_output_row_t *rows =

0 commit comments

Comments
 (0)