Skip to content

Commit 9ef1c23

Browse files
Andrey Shinkarenkoclaude
andcommitted
fix(search): index docstrings and accept non-ASCII query bytes
BM25 search was unusable for any project documented in a non-Latin script, for two independent reasons. bm25_build_match() treated only [a-zA-Z0-9_] as token bytes, so a Cyrillic (or Greek, or CJK) query was tokenized to nothing, reached FTS5 as an empty MATCH and returned unranked noise. Bytes >= 0x80 are now token bytes: no UTF-8 decoding is needed here, only keeping multi-byte sequences intact, since the unicode61 tokenizer already applies the real word-boundary rules on both sides. nodes_fts also indexed identifiers only. Identifiers are English in practice even when the team writes prose in another language, so the one field that could match a native-language query was the one field never indexed. A docstring column is added and populated from json_extract(properties, '$.docstring') in both backfill paths. Databases created before this carry the 4-column table, which CREATE ... IF NOT EXISTS would leave in place and every backfill would then fail on, so the new column is probed and the table dropped when missing — it is contentless, nothing is lost, the next backfill repopulates it from nodes. English search improves as a side effect: docstrings carry intent that identifiers alone do not. Fixes #1401 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent d6be58e commit 9ef1c23

5 files changed

Lines changed: 114 additions & 14 deletions

File tree

‎src/mcp/mcp.c‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2769,6 +2769,18 @@ static sqlite3_destructor_type mcp_sqlite_transient(void) {
27692769
}
27702770
#define MCP_SQLITE_TRANSIENT (mcp_sqlite_transient())
27712771

2772+
/* A byte that may sit inside a query token. ASCII alphanumerics and '_' as
2773+
* before, plus every byte >= 0x80: those are UTF-8 lead/continuation bytes, so
2774+
* treating them as separators (the previous behaviour) silently dropped every
2775+
* non-Latin word from the query — a Cyrillic, Greek or CJK search reached the
2776+
* index as an empty match and returned unranked noise. We do not need to
2777+
* decode UTF-8 here, only to keep multi-byte sequences intact: the FTS5
2778+
* unicode61 tokenizer applies the real word-boundary rules on both sides. */
2779+
static int bm25_is_token_byte(unsigned char c) {
2780+
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' ||
2781+
c >= 0x80;
2782+
}
2783+
27722784
static int bm25_build_match(const char *query, char *out, size_t out_size) {
27732785
if (!query || !out || out_size < BM25_MIN_BUF) {
27742786
return 0;
@@ -2777,16 +2789,14 @@ static int bm25_build_match(const char *query, char *out, size_t out_size) {
27772789
int tokens = 0;
27782790
const char *p = query;
27792791
while (*p) {
2780-
while (*p && !((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') ||
2781-
(*p >= '0' && *p <= '9') || *p == '_')) {
2792+
while (*p && !bm25_is_token_byte((unsigned char)*p)) {
27822793
p++;
27832794
}
27842795
if (!*p) {
27852796
break;
27862797
}
27872798
const char *tok_start = p;
2788-
while (*p && ((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') ||
2789-
(*p >= '0' && *p <= '9') || *p == '_')) {
2799+
while (*p && bm25_is_token_byte((unsigned char)*p)) {
27902800
p++;
27912801
}
27922802
size_t tok_len = (size_t)(p - tok_start);

‎src/pipeline/pipeline.c‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1418,16 +1418,26 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *fil
14181418
* Contentless FTS5 requires the special 'delete-all' command instead of
14191419
* DELETE FROM to wipe prior rows (there's no underlying content table).
14201420
* Falls back to plain names if cbm_camel_split is unavailable (which
1421-
* shouldn't happen because we always register it, but we stay defensive). */
1421+
* shouldn't happen because we always register it, but we stay defensive).
1422+
*
1423+
* `docstring` is pulled out of the properties JSON so prose in comments
1424+
* is searchable, not just identifiers. It matters most for non-English
1425+
* codebases: identifiers are almost always English, so a query written
1426+
* in the team's own language matched nothing at all before. The
1427+
* unicode61 tokenizer already handles non-Latin scripts. */
14221428
CBM_PROF_START(t_fts);
14231429
cbm_store_exec(hash_store, "INSERT INTO nodes_fts(nodes_fts) VALUES('delete-all');");
14241430
if (cbm_store_exec(hash_store,
1425-
"INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path) "
1426-
"SELECT id, cbm_camel_split(name), qualified_name, label, file_path "
1431+
"INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path, "
1432+
"docstring) "
1433+
"SELECT id, cbm_camel_split(name), qualified_name, label, file_path, "
1434+
"COALESCE(json_extract(properties, '$.docstring'), '') "
14271435
"FROM nodes;") != CBM_STORE_OK) {
14281436
cbm_store_exec(hash_store,
1429-
"INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path) "
1430-
"SELECT id, name, qualified_name, label, file_path FROM nodes;");
1437+
"INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path, "
1438+
"docstring) "
1439+
"SELECT id, name, qualified_name, label, file_path, "
1440+
"COALESCE(json_extract(properties, '$.docstring'), '') FROM nodes;");
14311441
}
14321442
CBM_PROF_END("persist", "5_fts_backfill", t_fts);
14331443

‎src/pipeline/pipeline_incremental.c‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -690,12 +690,16 @@ static int dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *p
690690
rc = CBM_STORE_ERR;
691691
}
692692
if (cbm_store_exec(hash_store,
693-
"INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path) "
694-
"SELECT id, cbm_camel_split(name), qualified_name, label, file_path "
693+
"INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path, "
694+
"docstring) "
695+
"SELECT id, cbm_camel_split(name), qualified_name, label, file_path, "
696+
"COALESCE(json_extract(properties, '$.docstring'), '') "
695697
"FROM nodes;") != CBM_STORE_OK) {
696698
if (cbm_store_exec(hash_store,
697-
"INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path) "
698-
"SELECT id, name, qualified_name, label, file_path FROM nodes;") !=
699+
"INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path, "
700+
"docstring) "
701+
"SELECT id, name, qualified_name, label, file_path, "
702+
"COALESCE(json_extract(properties, '$.docstring'), '') FROM nodes;") !=
699703
CBM_STORE_OK) {
700704
rc = CBM_STORE_ERR;
701705
}

‎src/store/store.c‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -337,9 +337,22 @@ static int init_schema(cbm_store_t *s) {
337337
* Fails silently if FTS5 is not compiled in (SQLITE_ENABLE_FTS5). */
338338
{
339339
char *fts_err = NULL;
340+
341+
/* Schema upgrade (#docstring-fts): databases created before the
342+
* `docstring` column exists carry a 4-column nodes_fts, and
343+
* CREATE ... IF NOT EXISTS would leave them that way — the backfill
344+
* INSERT would then fail on every index run. The table is
345+
* contentless, so dropping it loses no source data: the next
346+
* backfill repopulates it from `nodes`. Detect the old shape by
347+
* probing the new column and drop only on failure. */
348+
if (sqlite3_exec(s->db, "SELECT docstring FROM nodes_fts LIMIT 0;", NULL, NULL, NULL) !=
349+
SQLITE_OK) {
350+
sqlite3_exec(s->db, "DROP TABLE IF EXISTS nodes_fts;", NULL, NULL, NULL);
351+
}
352+
340353
int fts_rc = sqlite3_exec(s->db,
341354
"CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5("
342-
" name, qualified_name, label, file_path,"
355+
" name, qualified_name, label, file_path, docstring,"
343356
" content='',"
344357
" tokenize='unicode61 remove_diacritics 2'"
345358
");",

‎tests/test_mcp.c‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2203,6 +2203,68 @@ TEST(tool_search_graph_query_honors_file_pattern_issue552) {
22032203
PASS();
22042204
}
22052205

2206+
/* Identifiers are English almost everywhere, so a team that documents in its
2207+
* own language had no working full-text query: the docstring never reached the
2208+
* FTS index, and the query tokenizer dropped every non-ASCII byte on the way
2209+
* in. Both halves are exercised here — a Cyrillic query must find the node
2210+
* whose only Cyrillic text lives in its docstring. */
2211+
TEST(tool_search_graph_query_matches_non_ascii_docstring) {
2212+
cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);
2213+
ASSERT_NOT_NULL(srv);
2214+
cbm_store_t *st = cbm_mcp_server_store(srv);
2215+
ASSERT_NOT_NULL(st);
2216+
2217+
const char *proj = "non-ascii-doc";
2218+
cbm_mcp_server_set_project(srv, proj);
2219+
cbm_store_upsert_project(st, proj, "/tmp/non-ascii-doc");
2220+
2221+
cbm_node_t documented = {0};
2222+
documented.project = proj;
2223+
documented.label = "Function";
2224+
documented.name = "handle_payment";
2225+
documented.qualified_name = "non-ascii-doc.src.billing.handle_payment";
2226+
documented.file_path = "src/billing.c";
2227+
documented.start_line = 1;
2228+
documented.end_line = 9;
2229+
documented.properties_json = "{\"docstring\":\"Провести платёж по счёту\"}";
2230+
ASSERT_GT(cbm_store_upsert_node(st, &documented), 0);
2231+
2232+
cbm_node_t undocumented = {0};
2233+
undocumented.project = proj;
2234+
undocumented.label = "Function";
2235+
undocumented.name = "unrelated_helper";
2236+
undocumented.qualified_name = "non-ascii-doc.src.util.unrelated_helper";
2237+
undocumented.file_path = "src/util.c";
2238+
undocumented.start_line = 1;
2239+
undocumented.end_line = 3;
2240+
ASSERT_GT(cbm_store_upsert_node(st, &undocumented), 0);
2241+
2242+
cbm_store_exec(st, "INSERT INTO nodes_fts(nodes_fts) VALUES('delete-all');");
2243+
ASSERT_EQ(cbm_store_exec(st,
2244+
"INSERT INTO nodes_fts(rowid, name, qualified_name, label, "
2245+
"file_path, docstring) "
2246+
"SELECT id, cbm_camel_split(name), qualified_name, label, file_path, "
2247+
"COALESCE(json_extract(properties, '$.docstring'), '') "
2248+
"FROM nodes;"),
2249+
CBM_STORE_OK);
2250+
2251+
char *resp = cbm_mcp_server_handle(
2252+
srv, "{\"jsonrpc\":\"2.0\",\"id\":553,\"method\":\"tools/call\","
2253+
"\"params\":{\"name\":\"search_graph\","
2254+
"\"arguments\":{\"project\":\"non-ascii-doc\",\"query\":\"платёж\",\"limit\":10}}}");
2255+
ASSERT_NOT_NULL(resp);
2256+
char *inner = extract_text_content(resp);
2257+
ASSERT_NOT_NULL(inner);
2258+
ASSERT_NOT_NULL(strstr(inner, "search_mode: bm25"));
2259+
ASSERT_NOT_NULL(strstr(inner, "handle_payment"));
2260+
ASSERT_NULL(strstr(inner, "unrelated_helper"));
2261+
2262+
free(inner);
2263+
free(resp);
2264+
cbm_mcp_server_free(srv);
2265+
PASS();
2266+
}
2267+
22062268
/* Resource discovery methods this server doesn't populate must return EMPTY
22072269
* lists, not -32601 Method-not-found: clients like Cline probe them on connect
22082270
* and surface the errors as a failed connection (#958). */
@@ -10332,6 +10394,7 @@ SUITE(mcp) {
1033210394
RUN_TEST(tool_output_regression_gate);
1033310395
RUN_TEST(tool_output_byte_budgets);
1033410396
RUN_TEST(tool_search_graph_query_honors_file_pattern_issue552);
10397+
RUN_TEST(tool_search_graph_query_matches_non_ascii_docstring);
1033510398
RUN_TEST(mcp_resource_discovery_methods_return_empty_lists);
1033610399
RUN_TEST(tool_query_graph_basic);
1033710400
RUN_TEST(tool_index_status_no_project);

0 commit comments

Comments
 (0)