Skip to content

Commit ee9dc48

Browse files
committed
Merge origin/main into fix/issue-2265-unresolved-call-coverage
Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
2 parents 9c07572 + 1f934b6 commit ee9dc48

6 files changed

Lines changed: 589 additions & 22 deletions

File tree

‎internal/cbm/extract_url_fold.c‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -401,10 +401,22 @@ static bool is_interpolated_string(TSNode node) {
401401
return false;
402402
}
403403

404+
/* A Python string whose opening token is more than one quote -- a prefix
405+
* (r"", b"", u"", an f"" with no field) or a triple quote -- cannot go down
406+
* the literal path: that strips exactly one quote from each end, so the
407+
* prefix and the extra quotes stayed in the URL and it never matched a route
408+
* (#523). Folding reads the content pieces instead. JS/TS strings have no
409+
* string_start, so this is Python only. */
410+
static bool is_wrapped_string(TSNode node) {
411+
TSNode start = ts_node_child(node, 0);
412+
return !ts_node_is_null(start) && strcmp(ts_node_type(start), "string_start") == 0 &&
413+
ts_node_end_byte(start) - ts_node_start_byte(start) > SKIP_ONE;
414+
}
415+
404416
static bool is_foldable_arg(TSNode arg) {
405417
const char *k = ts_node_type(arg);
406418
if (strcmp(k, "string") == 0) {
407-
return is_interpolated_string(arg);
419+
return is_interpolated_string(arg) || is_wrapped_string(arg);
408420
}
409421
return strcmp(k, "template_string") == 0 || strcmp(k, "binary_expression") == 0 ||
410422
strcmp(k, "binary_operator") == 0 || strcmp(k, "identifier") == 0 ||

‎src/mcp/mcp.c‎

Lines changed: 138 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -4013,6 +4013,7 @@ enum {
40134013
BM25_BIND_EXACT = 8,
40144014
BM25_BIND_KIND = 9,
40154015
BM25_BIND_NS = 10,
4016+
BM25_BIND_LABEL_WEIGHT = 11,
40164017
BM25_SQL_AUTO_LEN = -1,
40174018
/* Inner FTS5 candidate cap. SQLite can early-terminate a plain FTS5 query
40184019
* (no JOIN/WHERE on outer table) of the form:
@@ -4021,6 +4022,11 @@ enum {
40214022
* and then joining/filtering/re-ranking those, we bound all work to O(N) where
40224023
* N = BM25_INNER_LIMIT rather than the full match set size. */
40234024
BM25_INNER_LIMIT = 2000,
4025+
/* Longest label spliced into the MATCH pre-filter; real labels are short
4026+
* identifiers, so anything longer simply skips the pre-filter. */
4027+
BM25_LABEL_MAX = 64,
4028+
/* Room for "(<tokens>) AND label : \"<label>\"" around the token list. */
4029+
BM25_MATCH_BUF = BM25_QUERY_BUF + BM25_LABEL_MAX + 32,
40244030
};
40254031

40264032
/* Column weights for nodes_fts (name, qualified_name, label, file_path, body).
@@ -4039,8 +4045,28 @@ enum {
40394045
* Safe against a legacy four-column nodes_fts: FTS5's bm25() reads a weight
40404046
* only when an instance actually lands in that column (`nVal > ic`), so the
40414047
* fifth weight is simply never consulted on a table that has no fifth
4042-
* column. */
4043-
#define BM25_WEIGHTS "bm25(nodes_fts, 1.0, 1.0, 1.0, 1.0, 0.3)"
4048+
* column.
4049+
*
4050+
* The label weight is bound (?11): 1.0 normally, 0.0 when the MATCH is scoped
4051+
* to a label (#2386). Every candidate then carries that label, so a label hit
4052+
* cannot tell them apart, and a zero weight keeps the added `label : "..."`
4053+
* phrase from shifting BM25 order by row length. */
4054+
#define BM25_WEIGHTS "bm25(nodes_fts, 1.0, 1.0, ?11, 1.0, 0.3)"
4055+
static const double BM25_LABEL_WEIGHT = 1.0;
4056+
static const double BM25_LABEL_WEIGHT_SCOPED = 0.0;
4057+
4058+
/* Labels an UNLABELLED query never ranks: path-shaped File/Folder nodes, the
4059+
* Project node and Variables match almost any token through their qualified
4060+
* name and file path, and would crowd definitions out of the result. They are
4061+
* still in nodes_fts, so an explicit `label` that names one of them is served
4062+
* from its own label-scoped window like any other label (?7 is the caller's
4063+
* label, NULL when none was given). Excluding them there too returned 0 rows
4064+
* -- often with an exact "eq" total -- although such nodes match. Section and
4065+
* Module are NOT excluded (#518/#519): they carry the prose the body column
4066+
* indexes. Shared by the ranked and the count query, so the two cannot
4067+
* desynchronise. */
4068+
#define BM25_UNLABELLED_EXCLUSION \
4069+
"(?7 IS NOT NULL OR n.label NOT IN ('File','Folder','Variable','Project'))"
40444070

40454071
/* Module-local SQLITE_TRANSIENT wrapper to dodge performance-no-int-to-ptr.
40464072
* See the matching helper in src/store/store.c for the same pattern. */
@@ -4091,6 +4117,85 @@ static int bm25_build_match(const char *query, char *out, size_t out_size) {
40914117
return tokens;
40924118
}
40934119

4120+
/* Scope an FTS5 MATCH expression to one node label (#2386). The label filter
4121+
* must choose the BM25_INNER_LIMIT candidate window, not trim it afterwards:
4122+
* otherwise a label that is rare among a common token's matches falls outside
4123+
* the window and the search returns 0 rows although matching nodes exist.
4124+
* FTS5 matching is case-insensitive, so this is a superset pre-filter; the
4125+
* caller keeps the exact `n.label = ?` check. Only a plain identifier is
4126+
* spliced in, as a quoted FTS5 string; any other label leaves `tokens`
4127+
* unscoped, which is the pre-#2386 behaviour. Returns the label-column weight
4128+
* to bind for the resulting MATCH. */
4129+
static double bm25_scope_match_to_label(const char *tokens, const char *label, char *out,
4130+
size_t out_size) {
4131+
snprintf(out, out_size, "%s", tokens);
4132+
if (!label || !label[0]) {
4133+
return BM25_LABEL_WEIGHT;
4134+
}
4135+
size_t len = 0;
4136+
for (; label[len]; len++) {
4137+
char ch = label[len];
4138+
bool ident = (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||
4139+
(ch >= '0' && ch <= '9') || ch == '_';
4140+
if (!ident || len >= BM25_LABEL_MAX) {
4141+
return BM25_LABEL_WEIGHT;
4142+
}
4143+
}
4144+
char scoped[BM25_MATCH_BUF];
4145+
int n = snprintf(scoped, sizeof(scoped), "(%s) AND label : \"%s\"", tokens, label);
4146+
if (n <= 0 || (size_t)n >= sizeof(scoped) || (size_t)n >= out_size) {
4147+
return BM25_LABEL_WEIGHT;
4148+
}
4149+
memcpy(out, scoped, (size_t)n + 1);
4150+
return BM25_LABEL_WEIGHT_SCOPED;
4151+
}
4152+
4153+
/* A BM25 result is empty although nodes match when the label is spelled in a
4154+
* case no node carries: labels are case-sensitive, so `label=class` finds
4155+
* nothing while Class nodes match the query. Returns a hint (memory core,
4156+
* CBM_MEM_CLASS_OTHER) naming the label this project actually holds, or NULL
4157+
* when the label exists exactly (a true zero). The exact probe is an index
4158+
* seek on (project, label); the case-insensitive scan runs only when it
4159+
* misses. The caller's label is never echoed back, only the stored spelling. */
4160+
static char *bm25_label_zero_hint(sqlite3 *db, const char *project, const char *label) {
4161+
enum { HINT_BIND_PROJECT = 1, HINT_BIND_LABEL = 2, HINT_COL_LABEL = 0, HINT_PROBE_EXACT = 0 };
4162+
if (!db || !label || !label[0]) {
4163+
return NULL;
4164+
}
4165+
static const char *const probes[] = {
4166+
"SELECT label FROM nodes WHERE project = ?1 AND label = ?2 LIMIT 1",
4167+
"SELECT label FROM nodes WHERE project = ?1 AND label = ?2 COLLATE NOCASE LIMIT 1",
4168+
};
4169+
char found[CBM_SZ_256] = "";
4170+
for (size_t i = 0; i < sizeof(probes) / sizeof(probes[0]) && !found[0]; i++) {
4171+
sqlite3_stmt *st = NULL;
4172+
if (sqlite3_prepare_v2(db, probes[i], BM25_SQL_AUTO_LEN, &st, NULL) != SQLITE_OK) {
4173+
return NULL;
4174+
}
4175+
sqlite3_bind_text(st, HINT_BIND_PROJECT, project, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4176+
sqlite3_bind_text(st, HINT_BIND_LABEL, label, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4177+
if (sqlite3_step(st) == SQLITE_ROW) {
4178+
const char *stored = (const char *)sqlite3_column_text(st, HINT_COL_LABEL);
4179+
snprintf(found, sizeof(found), "%s", stored ? stored : "");
4180+
}
4181+
sqlite3_finalize(st);
4182+
if (found[0] && i == HINT_PROBE_EXACT) {
4183+
return NULL;
4184+
}
4185+
}
4186+
if (!found[0]) {
4187+
return cbm_mem_strdup(CBM_MEM_CLASS_OTHER,
4188+
"no node in this project carries this label; get_graph_schema "
4189+
"lists the labels this index holds");
4190+
}
4191+
char hint[CBM_SZ_512];
4192+
snprintf(hint, sizeof(hint),
4193+
"no node carries this label as spelled; labels are case-sensitive -- retry with "
4194+
"label=\"%s\"",
4195+
found);
4196+
return cbm_mem_strdup(CBM_MEM_CLASS_OTHER, hint);
4197+
}
4198+
40944199
static char *bm25_file_pattern_like(const char *file_pattern) {
40954200
if (!file_pattern) {
40964201
return NULL;
@@ -4142,7 +4247,8 @@ static void bm25_lines_str(char *out, size_t size, int start, int end) {
41424247

41434248
static void bm25_emit_metadata_tree(cbm_sb_t *sb, int total, int offset, int returned,
41444249
bool candidate_window_saturated, bool budget_hit,
4145-
bool floor_exceeded, size_t max_output_bytes) {
4250+
bool floor_exceeded, size_t max_output_bytes,
4251+
const char *zero_hint) {
41464252
bool has_more = total > offset + returned;
41474253
cbm_tree_scalar_int(sb, "total", total);
41484254
cbm_tree_scalar_str(sb, "total_relation", candidate_window_saturated ? "gte" : "eq");
@@ -4173,12 +4279,15 @@ static void bm25_emit_metadata_tree(cbm_sb_t *sb, int total, int offset, int ret
41734279
cbm_tree_scalar_str(sb, "hint",
41744280
"first identity row exceeds max_output_bytes; raise "
41754281
"max_output_tokens");
4282+
} else if (zero_hint) {
4283+
cbm_tree_scalar_str(sb, "hint", zero_hint);
41764284
}
41774285
}
41784286

41794287
static void bm25_emit_metadata_json(yyjson_mut_doc *doc, yyjson_mut_val *root, int total,
41804288
int offset, int returned, bool candidate_window_saturated,
4181-
bool budget_hit, bool floor_exceeded, size_t max_output_bytes) {
4289+
bool budget_hit, bool floor_exceeded, size_t max_output_bytes,
4290+
const char *zero_hint) {
41824291
bool has_more = total > offset + returned;
41834292
yyjson_mut_obj_add_int(doc, root, "total", total);
41844293
yyjson_mut_obj_add_str(doc, root, "total_relation", candidate_window_saturated ? "gte" : "eq");
@@ -4210,12 +4319,14 @@ static void bm25_emit_metadata_json(yyjson_mut_doc *doc, yyjson_mut_val *root, i
42104319
yyjson_mut_obj_add_str(doc, root, "hint",
42114320
"first identity row exceeds max_output_bytes; raise "
42124321
"max_output_tokens");
4322+
} else if (zero_hint) {
4323+
yyjson_mut_obj_add_strcpy(doc, root, "hint", zero_hint);
42134324
}
42144325
}
42154326

42164327
static char *bm25_render(const bm25_output_row_t *rows, int returned, int total, int offset,
42174328
bool candidate_window_saturated, bool tree_format, bool budget_hit,
4218-
bool floor_exceeded, size_t max_output_bytes) {
4329+
bool floor_exceeded, size_t max_output_bytes, const char *zero_hint) {
42194330
if (tree_format) {
42204331
cbm_sb_t sb;
42214332
cbm_sb_init(&sb);
@@ -4233,7 +4344,7 @@ static char *bm25_render(const bm25_output_row_t *rows, int returned, int total,
42334344
cbm_tree_row_end(&sb);
42344345
}
42354346
bm25_emit_metadata_tree(&sb, total, offset, returned, candidate_window_saturated,
4236-
budget_hit, floor_exceeded, max_output_bytes);
4347+
budget_hit, floor_exceeded, max_output_bytes, zero_hint);
42374348
return cbm_sb_finish(&sb);
42384349
}
42394350

@@ -4260,7 +4371,7 @@ static char *bm25_render(const bm25_output_row_t *rows, int returned, int total,
42604371
}
42614372
yyjson_mut_obj_add_val(doc, root, "rows", result_rows);
42624373
bm25_emit_metadata_json(doc, root, total, offset, returned, candidate_window_saturated,
4263-
budget_hit, floor_exceeded, max_output_bytes);
4374+
budget_hit, floor_exceeded, max_output_bytes, zero_hint);
42644375
char *json = yy_doc_to_str(doc);
42654376
yyjson_mut_doc_free(doc);
42664377
return json;
@@ -4277,11 +4388,16 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
42774388
if (!db) {
42784389
return NULL;
42794390
}
4280-
char fts_query[BM25_QUERY_BUF];
4281-
int tok_count = bm25_build_match(query, fts_query, sizeof(fts_query));
4391+
char fts_tokens[BM25_QUERY_BUF];
4392+
int tok_count = bm25_build_match(query, fts_tokens, sizeof(fts_tokens));
42824393
if (tok_count == 0) {
42834394
return NULL;
42844395
}
4396+
/* One MATCH string for the ranked query, the count and the saturation
4397+
* probe, so all three describe the same label-scoped window. */
4398+
char fts_query[BM25_MATCH_BUF];
4399+
double label_weight =
4400+
bm25_scope_match_to_label(fts_tokens, label, fts_query, sizeof(fts_query));
42854401
char *file_like = bm25_file_pattern_like(file_pattern);
42864402

42874403
/* BM25 ranked query using a two-step approach to enable FTS5 early termination.
@@ -4324,12 +4440,7 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
43244440
") fts "
43254441
"JOIN nodes n ON n.id = fts.rowid "
43264442
"WHERE n.project = ?2 "
4327-
/* Section and Module are NO LONGER excluded (#518/#519): they are the
4328-
* labels that carry prose — a Markdown section's body, a config file's
4329-
* description — so excluding them made the body column unreachable.
4330-
* This exclusion list is MIRRORED in the count query below; the two
4331-
* must be changed together or results desynchronise from counts. */
4332-
" AND n.label NOT IN ('File','Folder','Variable','Project') "
4443+
" AND " BM25_UNLABELLED_EXCLUSION " "
43334444
" AND (?6 IS NULL OR n.file_path LIKE ?6) "
43344445
/* The caller's label filter applies in query mode exactly as it does
43354446
* in the structural mode (2026-09-16 probe: `label=Class` was ignored
@@ -4379,6 +4490,7 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
43794490
} else {
43804491
sqlite3_bind_null(stmt, BM25_BIND_NS);
43814492
}
4493+
sqlite3_bind_double(stmt, BM25_BIND_LABEL_WEIGHT, label_weight);
43824494

43834495
/* Count hits within the same inner-limit window — capped at BM25_INNER_LIMIT.
43844496
* Uses the identical subquery structure so the FTS5 early-exit applies here too. */
@@ -4395,7 +4507,7 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
43954507
/* MIRRORS the ranked query's filter verbatim — same weights, same
43964508
* label exclusions. Changing one alone reports a total that does
43974509
* not describe the rows returned. */
4398-
" AND n.label NOT IN ('File','Folder','Variable','Project')"
4510+
" AND " BM25_UNLABELLED_EXCLUSION " "
43994511
" AND (?6 IS NULL OR n.file_path LIKE ?6)"
44004512
" AND (?7 IS NULL OR n.label = ?7)"
44014513
" AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) "
@@ -4412,6 +4524,7 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
44124524
sqlite3_bind_text(cs, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN,
44134525
MCP_SQLITE_TRANSIENT);
44144526
sqlite3_bind_int(cs, BM25_BIND_LIMIT, BM25_INNER_LIMIT);
4527+
sqlite3_bind_double(cs, BM25_BIND_LABEL_WEIGHT, label_weight);
44154528
if (file_like) {
44164529
sqlite3_bind_text(cs, BM25_BIND_FILE, file_like, BM25_SQL_AUTO_LEN,
44174530
MCP_SQLITE_TRANSIENT);
@@ -4444,9 +4557,10 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
44444557

44454558
/* The top-candidate window is a performance ceiling, not an exact-total
44464559
* boundary. Probe one candidate beyond it so a broad query never presents
4447-
* a window-local count as the complete match count. This is global to the
4448-
* FTS table; saturation is therefore conservatively reported even when
4449-
* later project/path filters might discard the hidden candidates. */
4560+
* a window-local count as the complete match count. The probe shares the
4561+
* label-scoped MATCH but is otherwise global to the FTS table; saturation
4562+
* is therefore conservatively reported even when later project/path
4563+
* filters might discard the hidden candidates. */
44504564
bool candidate_window_saturated = true;
44514565
{
44524566
const char *probe_sql = "SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1 "
@@ -4484,13 +4598,15 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
44844598
if (!rows) {
44854599
return heap_strdup("out of memory");
44864600
}
4601+
/* Never a silent zero for a label the index spells differently. */
4602+
char *zero_hint = total == 0 ? bm25_label_zero_hint(db, project, label) : NULL;
44874603

44884604
int returned = row_count;
44894605
bool budget_hit = false;
44904606
char *payload = NULL;
44914607
for (;;) {
44924608
payload = bm25_render(rows, returned, total, offset, candidate_window_saturated,
4493-
tree_format, budget_hit, false, max_output_bytes);
4609+
tree_format, budget_hit, false, max_output_bytes, zero_hint);
44944610
if (!payload || strlen(payload) <= max_output_bytes || returned == 0) {
44954611
break;
44964612
}
@@ -4503,8 +4619,9 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
45034619
if (!payload || strlen(payload) > max_output_bytes || (row_count > 0 && returned == 0)) {
45044620
free(payload);
45054621
payload = bm25_render(rows, 0, total, offset, candidate_window_saturated, tree_format, true,
4506-
row_count > 0, max_output_bytes);
4622+
row_count > 0, max_output_bytes, zero_hint);
45074623
}
4624+
cbm_free(CBM_MEM_CLASS_OTHER, zero_hint);
45084625
bm25_output_rows_free(rows, row_count);
45094626
return payload;
45104627
}

‎src/pipeline/pipeline_incremental.c‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3754,6 +3754,17 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
37543754
cbm_store_count_edges(staging, project));
37553755
cbm_log_info("delta.committed_counts", "elapsed_ms", itoa_buf((int)elapsed_ms(t)));
37563756

3757+
/* The staging file started as a byte-for-byte clone of the previous
3758+
* generation (cbm_delta_stage_clone above), so its `projects` row still
3759+
* carries whichever root_path/indexed_at the LAST full rebuild wrote.
3760+
* Every other write in this function patches nodes/edges/hashes/surfaces
3761+
* for the CURRENT run's repo_path, so the project row must be refreshed
3762+
* to match, or a later root_path-keyed read (get_code_snippet and
3763+
* friends) resolves files against a directory this run never touched. */
3764+
if (cbm_store_upsert_project(staging, project, cbm_pipeline_repo_path(p)) != CBM_STORE_OK) {
3765+
goto out;
3766+
}
3767+
37573768
{
37583769
int index_mode = cbm_pipeline_get_mode(p);
37593770
cbm_pipeline_generation_t generation = {

0 commit comments

Comments
 (0)