@@ -4036,6 +4036,8 @@ enum {
40364036 * and then joining/filtering/re-ranking those, we bound all work to O(N) where
40374037 * N = BM25_INNER_LIMIT rather than the full match set size. */
40384038 BM25_INNER_LIMIT = 2000,
4039+ /* SQLite reads a negative LIMIT as "no limit": rank every candidate. */
4040+ BM25_INNER_ALL = -1,
40394041 /* Longest label spliced into the MATCH pre-filter; real labels are short
40404042 * identifiers, so anything longer simply skips the pre-filter. */
40414043 BM25_LABEL_MAX = 64,
@@ -4391,6 +4393,96 @@ static char *bm25_render(const bm25_output_row_t *rows, int returned, int total,
43914393 return json;
43924394}
43934395
4396+ /* The MATCH expression and the filters the ranked and the count query share.
4397+ * One binder for both keeps them on identical parameters. */
4398+ typedef struct {
4399+ const char *fts_query;
4400+ const char *project;
4401+ const char *file_like;
4402+ const char *label;
4403+ const char *kind;
4404+ const char *ns;
4405+ double label_weight;
4406+ } bm25_filters_t;
4407+
4408+ static void bm25_bind_opt_text(sqlite3_stmt *st, int idx, const char *value) {
4409+ if (value && value[0]) {
4410+ sqlite3_bind_text(st, idx, value, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4411+ } else {
4412+ sqlite3_bind_null(st, idx);
4413+ }
4414+ }
4415+
4416+ static void bm25_bind_filters(sqlite3_stmt *st, const bm25_filters_t *f) {
4417+ sqlite3_bind_text(st, BM25_BIND_QUERY, f->fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4418+ sqlite3_bind_text(st, BM25_BIND_PROJECT, f->project, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4419+ bm25_bind_opt_text(st, BM25_BIND_FILE, f->file_like);
4420+ bm25_bind_opt_text(st, BM25_BIND_LABEL, f->label);
4421+ bm25_bind_opt_text(st, BM25_BIND_KIND, f->kind);
4422+ bm25_bind_opt_text(st, BM25_BIND_NS, f->ns);
4423+ sqlite3_bind_double(st, BM25_BIND_LABEL_WEIGHT, f->label_weight);
4424+ }
4425+
4426+ /* The top-candidate window is a performance ceiling, not an exact-total
4427+ * boundary. Probe one candidate beyond it so a broad query never presents a
4428+ * window-local count as the complete match count. The probe shares the
4429+ * label-scoped MATCH but is otherwise global to the FTS table; saturation is
4430+ * therefore conservatively reported even when later project/path filters
4431+ * might discard the hidden candidates. */
4432+ static bool bm25_window_saturated(sqlite3 *db, const char *fts_query) {
4433+ enum { PROBE_BIND_OFFSET = 2 };
4434+ const char *probe_sql = "SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1 "
4435+ "ORDER BY bm25(nodes_fts), rowid LIMIT 1 OFFSET ?2";
4436+ sqlite3_stmt *probe = NULL;
4437+ if (sqlite3_prepare_v2(db, probe_sql, BM25_SQL_AUTO_LEN, &probe, NULL) != SQLITE_OK) {
4438+ return true;
4439+ }
4440+ sqlite3_bind_text(probe, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4441+ sqlite3_bind_int(probe, PROBE_BIND_OFFSET, BM25_INNER_LIMIT);
4442+ bool saturated = sqlite3_step(probe) != SQLITE_DONE;
4443+ sqlite3_finalize(probe);
4444+ return saturated;
4445+ }
4446+
4447+ /* Count the candidates that pass the filters among the top `inner_limit`
4448+ * (BM25_INNER_ALL: among all of them), with the ranked query's subquery shape
4449+ * so FTS5 drives the scan here too. */
4450+ static int bm25_count_candidates(sqlite3 *db, const bm25_filters_t *f, int inner_limit) {
4451+ const char *count_sql =
4452+ "SELECT COUNT(*) FROM ("
4453+ " SELECT fts.rowid FROM ("
4454+ " SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1"
4455+ " ORDER BY " BM25_WEIGHTS " LIMIT ?3"
4456+ " ) fts "
4457+ " JOIN nodes n ON n.id = fts.rowid "
4458+ " WHERE n.project = ?2 "
4459+ /* MIRRORS the ranked query's filter verbatim — same weights, same
4460+ * label exclusions. Changing one alone reports a total that does
4461+ * not describe the rows returned. */
4462+ " AND " BM25_UNLABELLED_EXCLUSION " "
4463+ " AND (?6 IS NULL OR n.file_path LIKE ?6)"
4464+ " AND (?7 IS NULL OR n.label = ?7)"
4465+ " AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) "
4466+ " THEN json_extract(n.properties,'$.kind') END) = ?9)"
4467+ " AND (?10 IS NULL OR (CASE WHEN json_valid(n.properties) "
4468+ " THEN json_extract(n.properties,'$.obj_key') END) "
4469+ " LIKE ('%/%/' || replace(replace(replace(?10,'\\','\\\\'),'%','\\%'),"
4470+ " '_','\\_') || '/%') ESCAPE '\\')"
4471+ ")";
4472+ int total = 0;
4473+ sqlite3_stmt *cs = NULL;
4474+ if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) != SQLITE_OK) {
4475+ return 0;
4476+ }
4477+ bm25_bind_filters(cs, f);
4478+ sqlite3_bind_int(cs, BM25_BIND_LIMIT, inner_limit);
4479+ if (sqlite3_step(cs) == SQLITE_ROW) {
4480+ total = sqlite3_column_int(cs, 0);
4481+ }
4482+ sqlite3_finalize(cs);
4483+ return total;
4484+ }
4485+
43944486/* Run the BM25 full-text search path and return the JSON result string.
43954487 * Returns NULL if FTS5 is unavailable or the query produced no usable tokens,
43964488 * in which case the caller falls back to the regex-based search path. */
@@ -4478,117 +4570,39 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
44784570 free(file_like);
44794571 return NULL;
44804572 }
4481- sqlite3_bind_text(stmt, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4482- sqlite3_bind_text(stmt, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4573+ const bm25_filters_t filters = {.fts_query = fts_query,
4574+ .project = project,
4575+ .file_like = file_like,
4576+ .label = label,
4577+ .kind = kind,
4578+ .ns = ns,
4579+ .label_weight = label_weight};
4580+ bm25_bind_filters(stmt, &filters);
44834581 sqlite3_bind_int(stmt, BM25_BIND_LIMIT, limit > 0 ? limit : BM25_DEFAULT_LIMIT);
44844582 sqlite3_bind_int(stmt, BM25_BIND_OFFSET, offset > 0 ? offset : 0);
4485- sqlite3_bind_int(stmt, BM25_BIND_INNER, BM25_INNER_LIMIT);
4486- if (file_like) {
4487- sqlite3_bind_text(stmt, BM25_BIND_FILE, file_like, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4488- } else {
4489- sqlite3_bind_null(stmt, BM25_BIND_FILE);
4490- }
4491- if (label && label[0]) {
4492- sqlite3_bind_text(stmt, BM25_BIND_LABEL, label, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4493- } else {
4494- sqlite3_bind_null(stmt, BM25_BIND_LABEL);
4495- }
44964583 sqlite3_bind_text(stmt, BM25_BIND_EXACT, query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4497- if (kind && kind[0]) {
4498- sqlite3_bind_text(stmt, BM25_BIND_KIND, kind, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4499- } else {
4500- sqlite3_bind_null(stmt, BM25_BIND_KIND);
4501- }
4502- if (ns && ns[0]) {
4503- sqlite3_bind_text(stmt, BM25_BIND_NS, ns, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4504- } else {
4505- sqlite3_bind_null(stmt, BM25_BIND_NS);
4506- }
4507- sqlite3_bind_double(stmt, BM25_BIND_LABEL_WEIGHT, label_weight);
45084584
4509- /* Count hits within the same inner-limit window — capped at BM25_INNER_LIMIT.
4510- * Uses the identical subquery structure so the FTS5 early-exit applies here too. */
4511- int total = 0;
4512- {
4513- const char *count_sql =
4514- "SELECT COUNT(*) FROM ("
4515- " SELECT fts.rowid FROM ("
4516- " SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1"
4517- " ORDER BY " BM25_WEIGHTS " LIMIT ?3"
4518- " ) fts "
4519- " JOIN nodes n ON n.id = fts.rowid "
4520- " WHERE n.project = ?2 "
4521- /* MIRRORS the ranked query's filter verbatim — same weights, same
4522- * label exclusions. Changing one alone reports a total that does
4523- * not describe the rows returned. */
4524- " AND " BM25_UNLABELLED_EXCLUSION " "
4525- " AND (?6 IS NULL OR n.file_path LIKE ?6)"
4526- " AND (?7 IS NULL OR n.label = ?7)"
4527- " AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) "
4528- " THEN json_extract(n.properties,'$.kind') END) = ?9)"
4529- " AND (?10 IS NULL OR (CASE WHEN json_valid(n.properties) "
4530- " THEN json_extract(n.properties,'$.obj_key') END) "
4531- " LIKE ('%/%/' || replace(replace(replace(?10,'\\','\\\\'),'%','\\%'),"
4532- " '_','\\_') || '/%') ESCAPE '\\')"
4533- ")";
4534- sqlite3_stmt *cs = NULL;
4535- if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) == SQLITE_OK) {
4536- sqlite3_bind_text(cs, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN,
4537- MCP_SQLITE_TRANSIENT);
4538- sqlite3_bind_text(cs, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN,
4539- MCP_SQLITE_TRANSIENT);
4540- sqlite3_bind_int(cs, BM25_BIND_LIMIT, BM25_INNER_LIMIT);
4541- sqlite3_bind_double(cs, BM25_BIND_LABEL_WEIGHT, label_weight);
4542- if (file_like) {
4543- sqlite3_bind_text(cs, BM25_BIND_FILE, file_like, BM25_SQL_AUTO_LEN,
4544- MCP_SQLITE_TRANSIENT);
4545- } else {
4546- sqlite3_bind_null(cs, BM25_BIND_FILE);
4547- }
4548- if (label && label[0]) {
4549- sqlite3_bind_text(cs, BM25_BIND_LABEL, label, BM25_SQL_AUTO_LEN,
4550- MCP_SQLITE_TRANSIENT);
4551- } else {
4552- sqlite3_bind_null(cs, BM25_BIND_LABEL);
4553- }
4554- if (kind && kind[0]) {
4555- sqlite3_bind_text(cs, BM25_BIND_KIND, kind, BM25_SQL_AUTO_LEN,
4556- MCP_SQLITE_TRANSIENT);
4557- } else {
4558- sqlite3_bind_null(cs, BM25_BIND_KIND);
4559- }
4560- if (ns && ns[0]) {
4561- sqlite3_bind_text(cs, BM25_BIND_NS, ns, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4562- } else {
4563- sqlite3_bind_null(cs, BM25_BIND_NS);
4564- }
4565- if (sqlite3_step(cs) == SQLITE_ROW) {
4566- total = sqlite3_column_int(cs, 0);
4567- }
4568- sqlite3_finalize(cs);
4569- }
4570- }
4571-
4572- /* The top-candidate window is a performance ceiling, not an exact-total
4573- * boundary. Probe one candidate beyond it so a broad query never presents
4574- * a window-local count as the complete match count. The probe shares the
4575- * label-scoped MATCH but is otherwise global to the FTS table; saturation
4576- * is therefore conservatively reported even when later project/path
4577- * filters might discard the hidden candidates. */
4578- bool candidate_window_saturated = true;
4579- {
4580- const char *probe_sql = "SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1 "
4581- "ORDER BY bm25(nodes_fts), rowid LIMIT 1 OFFSET ?2";
4582- sqlite3_stmt *probe = NULL;
4583- if (sqlite3_prepare_v2(db, probe_sql, BM25_SQL_AUTO_LEN, &probe, NULL) == SQLITE_OK) {
4584- sqlite3_bind_text(probe, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN,
4585- MCP_SQLITE_TRANSIENT);
4586- sqlite3_bind_int(probe, BM25_BIND_PROJECT, BM25_INNER_LIMIT);
4587- int probe_step = sqlite3_step(probe);
4588- candidate_window_saturated = probe_step != SQLITE_DONE;
4589- sqlite3_finalize(probe);
4590- }
4591- }
4585+ /* file_pattern, kind and namespace cannot be put into the MATCH the way
4586+ * the label is, so they filter AFTER the candidate window. When more
4587+ * candidates exist beyond it, a narrow filter can find none of its rows in
4588+ * the top BM25_INNER_LIMIT and the search answered 0 rows although
4589+ * matching nodes exist. A saturated window under such a filter -- or one
4590+ * that held no row passing the filters at all -- is therefore ranked over
4591+ * every candidate instead: the FTS-first subquery plan stays, the extra
4592+ * work is linear in the token's match count (the saturation probe already
4593+ * orders every match), and the total becomes exact. The decision depends
4594+ * only on the query, never on the page, so offset pages stay consistent.
4595+ * Unfiltered and label-only queries that find rows keep the window. */
4596+ bool saturated = bm25_window_saturated(db, fts_query);
4597+ bool post_window_filter = file_like || (kind && kind[0]) || (ns && ns[0]);
4598+ int inner_limit = saturated && post_window_filter ? BM25_INNER_ALL : BM25_INNER_LIMIT;
4599+ int total = bm25_count_candidates(db, &filters, inner_limit);
4600+ if (saturated && total == 0 && inner_limit != BM25_INNER_ALL) {
4601+ inner_limit = BM25_INNER_ALL;
4602+ total = bm25_count_candidates(db, &filters, inner_limit);
4603+ }
4604+ bool candidate_window_saturated = saturated && inner_limit != BM25_INNER_ALL;
4605+ sqlite3_bind_int(stmt, BM25_BIND_INNER, inner_limit);
45924606
45934607 int row_cap = limit > 0 ? limit : BM25_DEFAULT_LIMIT;
45944608 bm25_output_row_t *rows =
0 commit comments