@@ -4019,6 +4019,8 @@ enum {
40194019 * and then joining/filtering/re-ranking those, we bound all work to O(N) where
40204020 * N = BM25_INNER_LIMIT rather than the full match set size. */
40214021 BM25_INNER_LIMIT = 2000,
4022+ /* SQLite reads a negative LIMIT as "no limit": rank every candidate. */
4023+ BM25_INNER_ALL = -1,
40224024 /* Longest label spliced into the MATCH pre-filter; real labels are short
40234025 * identifiers, so anything longer simply skips the pre-filter. */
40244026 BM25_LABEL_MAX = 64,
@@ -4374,6 +4376,96 @@ static char *bm25_render(const bm25_output_row_t *rows, int returned, int total,
43744376 return json;
43754377}
43764378
4379+ /* The MATCH expression and the filters the ranked and the count query share.
4380+ * One binder for both keeps them on identical parameters. */
4381+ typedef struct {
4382+ const char *fts_query;
4383+ const char *project;
4384+ const char *file_like;
4385+ const char *label;
4386+ const char *kind;
4387+ const char *ns;
4388+ double label_weight;
4389+ } bm25_filters_t;
4390+
4391+ static void bm25_bind_opt_text(sqlite3_stmt *st, int idx, const char *value) {
4392+ if (value && value[0]) {
4393+ sqlite3_bind_text(st, idx, value, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4394+ } else {
4395+ sqlite3_bind_null(st, idx);
4396+ }
4397+ }
4398+
4399+ static void bm25_bind_filters(sqlite3_stmt *st, const bm25_filters_t *f) {
4400+ sqlite3_bind_text(st, BM25_BIND_QUERY, f->fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4401+ sqlite3_bind_text(st, BM25_BIND_PROJECT, f->project, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4402+ bm25_bind_opt_text(st, BM25_BIND_FILE, f->file_like);
4403+ bm25_bind_opt_text(st, BM25_BIND_LABEL, f->label);
4404+ bm25_bind_opt_text(st, BM25_BIND_KIND, f->kind);
4405+ bm25_bind_opt_text(st, BM25_BIND_NS, f->ns);
4406+ sqlite3_bind_double(st, BM25_BIND_LABEL_WEIGHT, f->label_weight);
4407+ }
4408+
4409+ /* The top-candidate window is a performance ceiling, not an exact-total
4410+ * boundary. Probe one candidate beyond it so a broad query never presents a
4411+ * window-local count as the complete match count. The probe shares the
4412+ * label-scoped MATCH but is otherwise global to the FTS table; saturation is
4413+ * therefore conservatively reported even when later project/path filters
4414+ * might discard the hidden candidates. */
4415+ static bool bm25_window_saturated(sqlite3 *db, const char *fts_query) {
4416+ enum { PROBE_BIND_OFFSET = 2 };
4417+ const char *probe_sql = "SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1 "
4418+ "ORDER BY bm25(nodes_fts), rowid LIMIT 1 OFFSET ?2";
4419+ sqlite3_stmt *probe = NULL;
4420+ if (sqlite3_prepare_v2(db, probe_sql, BM25_SQL_AUTO_LEN, &probe, NULL) != SQLITE_OK) {
4421+ return true;
4422+ }
4423+ sqlite3_bind_text(probe, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4424+ sqlite3_bind_int(probe, PROBE_BIND_OFFSET, BM25_INNER_LIMIT);
4425+ bool saturated = sqlite3_step(probe) != SQLITE_DONE;
4426+ sqlite3_finalize(probe);
4427+ return saturated;
4428+ }
4429+
4430+ /* Count the candidates that pass the filters among the top `inner_limit`
4431+ * (BM25_INNER_ALL: among all of them), with the ranked query's subquery shape
4432+ * so FTS5 drives the scan here too. */
4433+ static int bm25_count_candidates(sqlite3 *db, const bm25_filters_t *f, int inner_limit) {
4434+ const char *count_sql =
4435+ "SELECT COUNT(*) FROM ("
4436+ " SELECT fts.rowid FROM ("
4437+ " SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1"
4438+ " ORDER BY " BM25_WEIGHTS " LIMIT ?3"
4439+ " ) fts "
4440+ " JOIN nodes n ON n.id = fts.rowid "
4441+ " WHERE n.project = ?2 "
4442+ /* MIRRORS the ranked query's filter verbatim — same weights, same
4443+ * label exclusions. Changing one alone reports a total that does
4444+ * not describe the rows returned. */
4445+ " AND " BM25_UNLABELLED_EXCLUSION " "
4446+ " AND (?6 IS NULL OR n.file_path LIKE ?6)"
4447+ " AND (?7 IS NULL OR n.label = ?7)"
4448+ " AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) "
4449+ " THEN json_extract(n.properties,'$.kind') END) = ?9)"
4450+ " AND (?10 IS NULL OR (CASE WHEN json_valid(n.properties) "
4451+ " THEN json_extract(n.properties,'$.obj_key') END) "
4452+ " LIKE ('%/%/' || replace(replace(replace(?10,'\\','\\\\'),'%','\\%'),"
4453+ " '_','\\_') || '/%') ESCAPE '\\')"
4454+ ")";
4455+ int total = 0;
4456+ sqlite3_stmt *cs = NULL;
4457+ if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) != SQLITE_OK) {
4458+ return 0;
4459+ }
4460+ bm25_bind_filters(cs, f);
4461+ sqlite3_bind_int(cs, BM25_BIND_LIMIT, inner_limit);
4462+ if (sqlite3_step(cs) == SQLITE_ROW) {
4463+ total = sqlite3_column_int(cs, 0);
4464+ }
4465+ sqlite3_finalize(cs);
4466+ return total;
4467+ }
4468+
43774469/* Run the BM25 full-text search path and return the JSON result string.
43784470 * Returns NULL if FTS5 is unavailable or the query produced no usable tokens,
43794471 * in which case the caller falls back to the regex-based search path. */
@@ -4461,117 +4553,39 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
44614553 free(file_like);
44624554 return NULL;
44634555 }
4464- sqlite3_bind_text(stmt, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4465- sqlite3_bind_text(stmt, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4556+ const bm25_filters_t filters = {.fts_query = fts_query,
4557+ .project = project,
4558+ .file_like = file_like,
4559+ .label = label,
4560+ .kind = kind,
4561+ .ns = ns,
4562+ .label_weight = label_weight};
4563+ bm25_bind_filters(stmt, &filters);
44664564 sqlite3_bind_int(stmt, BM25_BIND_LIMIT, limit > 0 ? limit : BM25_DEFAULT_LIMIT);
44674565 sqlite3_bind_int(stmt, BM25_BIND_OFFSET, offset > 0 ? offset : 0);
4468- sqlite3_bind_int(stmt, BM25_BIND_INNER, BM25_INNER_LIMIT);
4469- if (file_like) {
4470- sqlite3_bind_text(stmt, BM25_BIND_FILE, file_like, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4471- } else {
4472- sqlite3_bind_null(stmt, BM25_BIND_FILE);
4473- }
4474- if (label && label[0]) {
4475- sqlite3_bind_text(stmt, BM25_BIND_LABEL, label, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4476- } else {
4477- sqlite3_bind_null(stmt, BM25_BIND_LABEL);
4478- }
44794566 sqlite3_bind_text(stmt, BM25_BIND_EXACT, query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4480- if (kind && kind[0]) {
4481- sqlite3_bind_text(stmt, BM25_BIND_KIND, kind, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4482- } else {
4483- sqlite3_bind_null(stmt, BM25_BIND_KIND);
4484- }
4485- if (ns && ns[0]) {
4486- sqlite3_bind_text(stmt, BM25_BIND_NS, ns, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4487- } else {
4488- sqlite3_bind_null(stmt, BM25_BIND_NS);
4489- }
4490- sqlite3_bind_double(stmt, BM25_BIND_LABEL_WEIGHT, label_weight);
4491-
4492- /* Count hits within the same inner-limit window — capped at BM25_INNER_LIMIT.
4493- * Uses the identical subquery structure so the FTS5 early-exit applies here too. */
4494- int total = 0;
4495- {
4496- const char *count_sql =
4497- "SELECT COUNT(*) FROM ("
4498- " SELECT fts.rowid FROM ("
4499- " SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1"
4500- " ORDER BY " BM25_WEIGHTS " LIMIT ?3"
4501- " ) fts "
4502- " JOIN nodes n ON n.id = fts.rowid "
4503- " WHERE n.project = ?2 "
4504- /* MIRRORS the ranked query's filter verbatim — same weights, same
4505- * label exclusions. Changing one alone reports a total that does
4506- * not describe the rows returned. */
4507- " AND " BM25_UNLABELLED_EXCLUSION " "
4508- " AND (?6 IS NULL OR n.file_path LIKE ?6)"
4509- " AND (?7 IS NULL OR n.label = ?7)"
4510- " AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) "
4511- " THEN json_extract(n.properties,'$.kind') END) = ?9)"
4512- " AND (?10 IS NULL OR (CASE WHEN json_valid(n.properties) "
4513- " THEN json_extract(n.properties,'$.obj_key') END) "
4514- " LIKE ('%/%/' || replace(replace(replace(?10,'\\','\\\\'),'%','\\%'),"
4515- " '_','\\_') || '/%') ESCAPE '\\')"
4516- ")";
4517- sqlite3_stmt *cs = NULL;
4518- if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) == SQLITE_OK) {
4519- sqlite3_bind_text(cs, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN,
4520- MCP_SQLITE_TRANSIENT);
4521- sqlite3_bind_text(cs, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN,
4522- MCP_SQLITE_TRANSIENT);
4523- sqlite3_bind_int(cs, BM25_BIND_LIMIT, BM25_INNER_LIMIT);
4524- sqlite3_bind_double(cs, BM25_BIND_LABEL_WEIGHT, label_weight);
4525- if (file_like) {
4526- sqlite3_bind_text(cs, BM25_BIND_FILE, file_like, BM25_SQL_AUTO_LEN,
4527- MCP_SQLITE_TRANSIENT);
4528- } else {
4529- sqlite3_bind_null(cs, BM25_BIND_FILE);
4530- }
4531- if (label && label[0]) {
4532- sqlite3_bind_text(cs, BM25_BIND_LABEL, label, BM25_SQL_AUTO_LEN,
4533- MCP_SQLITE_TRANSIENT);
4534- } else {
4535- sqlite3_bind_null(cs, BM25_BIND_LABEL);
4536- }
4537- if (kind && kind[0]) {
4538- sqlite3_bind_text(cs, BM25_BIND_KIND, kind, BM25_SQL_AUTO_LEN,
4539- MCP_SQLITE_TRANSIENT);
4540- } else {
4541- sqlite3_bind_null(cs, BM25_BIND_KIND);
4542- }
4543- if (ns && ns[0]) {
4544- sqlite3_bind_text(cs, BM25_BIND_NS, ns, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4545- } else {
4546- sqlite3_bind_null(cs, BM25_BIND_NS);
4547- }
4548- if (sqlite3_step(cs) == SQLITE_ROW) {
4549- total = sqlite3_column_int(cs, 0);
4550- }
4551- sqlite3_finalize(cs);
4552- }
4553- }
45544567
4555- /* The top-candidate window is a performance ceiling, not an exact-total
4556- * boundary. Probe one candidate beyond it so a broad query never presents
4557- * a window-local count as the complete match count. The probe shares the
4558- * label-scoped MATCH but is otherwise global to the FTS table; saturation
4559- * is therefore conservatively reported even when later project/path
4560- * filters might discard the hidden candidates. */
4561- bool candidate_window_saturated = true;
4562- {
4563- const char *probe_sql = "SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1 "
4564- "ORDER BY bm25(nodes_fts), rowid LIMIT 1 OFFSET ?2";
4565- sqlite3_stmt *probe = NULL;
4566- if (sqlite3_prepare_v2(db, probe_sql, BM25_SQL_AUTO_LEN, &probe, NULL) == SQLITE_OK) {
4567- sqlite3_bind_text(probe, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN,
4568- MCP_SQLITE_TRANSIENT);
4569- sqlite3_bind_int(probe, BM25_BIND_PROJECT, BM25_INNER_LIMIT);
4570- int probe_step = sqlite3_step(probe);
4571- candidate_window_saturated = probe_step != SQLITE_DONE;
4572- sqlite3_finalize(probe);
4573- }
4574- }
4568+ /* file_pattern, kind and namespace cannot be put into the MATCH the way
4569+ * the label is, so they filter AFTER the candidate window. When more
4570+ * candidates exist beyond it, a narrow filter can find none of its rows in
4571+ * the top BM25_INNER_LIMIT and the search answered 0 rows although
4572+ * matching nodes exist. A saturated window under such a filter -- or one
4573+ * that held no row passing the filters at all -- is therefore ranked over
4574+ * every candidate instead: the FTS-first subquery plan stays, the extra
4575+ * work is linear in the token's match count (the saturation probe already
4576+ * orders every match), and the total becomes exact. The decision depends
4577+ * only on the query, never on the page, so offset pages stay consistent.
4578+ * Unfiltered and label-only queries that find rows keep the window. */
4579+ bool saturated = bm25_window_saturated(db, fts_query);
4580+ bool post_window_filter = file_like || (kind && kind[0]) || (ns && ns[0]);
4581+ int inner_limit = saturated && post_window_filter ? BM25_INNER_ALL : BM25_INNER_LIMIT;
4582+ int total = bm25_count_candidates(db, &filters, inner_limit);
4583+ if (saturated && total == 0 && inner_limit != BM25_INNER_ALL) {
4584+ inner_limit = BM25_INNER_ALL;
4585+ total = bm25_count_candidates(db, &filters, inner_limit);
4586+ }
4587+ bool candidate_window_saturated = saturated && inner_limit != BM25_INNER_ALL;
4588+ sqlite3_bind_int(stmt, BM25_BIND_INNER, inner_limit);
45754589
45764590 int row_cap = limit > 0 ? limit : BM25_DEFAULT_LIMIT;
45774591 bm25_output_row_t *rows =
0 commit comments