Skip to content

Commit 05574ff

Browse files
committed
fix: preserve UTF-8 in search_code source
Signed-off-by: anfedoro <anfedoro@gmail.com>
1 parent 97ce23f commit 05574ff

2 files changed

Lines changed: 90 additions & 17 deletions

File tree

‎src/mcp/mcp.c‎

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -8287,16 +8287,6 @@ static char *handle_get_code_snippet(cbm_mcp_server_t *srv, const char *args) {
82878287

82888288
/* ── search_code v2: graph-augmented code search ─────────────── */
82898289

8290-
/* Strip non-ASCII bytes to guarantee valid UTF-8 JSON output */
8291-
enum { ASCII_MAX = 127 };
8292-
static void sanitize_ascii(char *s) {
8293-
for (unsigned char *p = (unsigned char *)s; *p; p++) {
8294-
if (*p > ASCII_MAX) {
8295-
*p = '?';
8296-
}
8297-
}
8298-
}
8299-
83008290
/* Intermediate grep match */
83018291
typedef struct {
83028292
char file[CBM_SZ_512];
@@ -8497,8 +8487,11 @@ static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, sear
84978487
}
84988488
char *source = read_file_lines(abs_path, s, e);
84998489
if (source) {
8500-
sanitize_ascii(source);
8501-
yyjson_mut_obj_add_strcpy(doc, item, "source", source);
8490+
char *safe_source = sanitize_utf8_lossy(source);
8491+
if (safe_source) {
8492+
yyjson_mut_obj_add_strcpy(doc, item, "source", safe_source);
8493+
free(safe_source);
8494+
}
85028495
free(source);
85038496
if (truncated) {
85048497
yyjson_mut_obj_add_int(doc, item, "source_start", s);
@@ -8513,8 +8506,11 @@ static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, sear
85138506
}
85148507
char *ctx = read_file_lines(abs_path, ctx_start, ctx_end);
85158508
if (ctx) {
8516-
sanitize_ascii(ctx);
8517-
yyjson_mut_obj_add_strcpy(doc, item, "context", ctx);
8509+
char *safe_context = sanitize_utf8_lossy(ctx);
8510+
if (safe_context) {
8511+
yyjson_mut_obj_add_strcpy(doc, item, "context", safe_context);
8512+
free(safe_context);
8513+
}
85188514
yyjson_mut_obj_add_int(doc, item, "context_start", ctx_start);
85198515
free(ctx);
85208516
}
@@ -8783,7 +8779,11 @@ static char *assemble_search_output(search_result_t *sr, int sr_count, grep_matc
87838779

87848780
char *json = yy_doc_to_str(doc);
87858781
if (json) {
8786-
sanitize_ascii(json);
8782+
char *safe_json = sanitize_utf8_lossy(json);
8783+
if (safe_json) {
8784+
free(json);
8785+
json = safe_json;
8786+
}
87878787
}
87888788
yyjson_mut_doc_free(doc);
87898789

@@ -8854,8 +8854,10 @@ static grep_match_t *collect_grep_matches(FILE *fp, const char *root_path, size_
88548854
safe_grow(gm, gm_count, gm_cap, PAIR_LEN);
88558855
snprintf(gm[gm_count].file, sizeof(gm[0].file), "%s", file);
88568856
gm[gm_count].line = (int)strtol(sep1 + SKIP_ONE, NULL, CBM_DECIMAL_BASE);
8857-
snprintf(gm[gm_count].content, sizeof(gm[0].content), "%s", sep2 + SKIP_ONE);
8858-
sanitize_ascii(gm[gm_count].content);
8857+
char *safe_content = sanitize_utf8_lossy(sep2 + SKIP_ONE);
8858+
snprintf(gm[gm_count].content, sizeof(gm[0].content), "%s",
8859+
safe_content ? safe_content : sep2 + SKIP_ONE);
8860+
free(safe_content);
88598861
gm_count++;
88608862
}
88618863

‎tests/test_mcp.c‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3530,6 +3530,76 @@ TEST(search_code_multi_word) {
35303530
PASS();
35313531
}
35323532

3533+
/* Regression guard: search_code full results must preserve valid UTF-8 source. */
3534+
TEST(search_code_full_preserves_utf8_source) {
3535+
char tmp[512];
3536+
snprintf(tmp, sizeof(tmp), "/tmp/cbm_srch_utf8_XXXXXX");
3537+
ASSERT_TRUE(cbm_mkdtemp(tmp) != NULL);
3538+
3539+
char project_dir[640];
3540+
snprintf(project_dir, sizeof(project_dir), "%s/project", tmp);
3541+
ASSERT_EQ(cbm_mkdir(project_dir), 0);
3542+
char design_dir[768];
3543+
snprintf(design_dir, sizeof(design_dir), "%s/design", project_dir);
3544+
ASSERT_EQ(cbm_mkdir(design_dir), 0);
3545+
3546+
char source_path[768];
3547+
snprintf(source_path, sizeof(source_path), "%s/design.md", design_dir);
3548+
FILE *fp = cbm_fopen(source_path, "wb");
3549+
ASSERT_NOT_NULL(fp);
3550+
const char source[] = "# accounting-design\nРусский текст: бухгалтерский учет.\n";
3551+
ASSERT_EQ(fwrite(source, 1, sizeof(source) - SKIP_ONE, fp), sizeof(source) - SKIP_ONE);
3552+
ASSERT_EQ(fclose(fp), 0);
3553+
3554+
cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);
3555+
ASSERT_NOT_NULL(srv);
3556+
cbm_store_t *st = cbm_mcp_server_store(srv);
3557+
ASSERT_NOT_NULL(st);
3558+
const char *project = "utf8-search";
3559+
cbm_mcp_server_set_project(srv, project);
3560+
cbm_store_upsert_project(st, project, project_dir);
3561+
3562+
cbm_node_t section = {.project = project,
3563+
.label = "Section",
3564+
.name = "accounting-design",
3565+
.qualified_name = "utf8-search.design.accounting-design",
3566+
.file_path = "design/design.md",
3567+
.start_line = 1,
3568+
.end_line = 2};
3569+
ASSERT_GT(cbm_store_upsert_node(st, &section), 0);
3570+
3571+
char *resp = cbm_mcp_server_handle(
3572+
srv, "{\"jsonrpc\":\"2.0\",\"id\":97,\"method\":\"tools/call\","
3573+
"\"params\":{\"name\":\"search_code\",\"arguments\":{"
3574+
"\"project\":\"utf8-search\",\"pattern\":\"accounting-design\","
3575+
"\"file_pattern\":\"*.md\",\"path_filter\":\"^design/\","
3576+
"\"mode\":\"full\",\"format\":\"json\",\"limit\":5}}}");
3577+
ASSERT_NOT_NULL(resp);
3578+
char *inner = extract_text_content(resp);
3579+
ASSERT_NOT_NULL(inner);
3580+
3581+
yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0);
3582+
ASSERT_NOT_NULL(doc);
3583+
yyjson_val *rows = yyjson_obj_get(yyjson_doc_get_root(doc), "rows");
3584+
ASSERT_NOT_NULL(rows);
3585+
ASSERT_TRUE(yyjson_arr_size(rows) > 0);
3586+
yyjson_val *row = yyjson_arr_get(rows, 0);
3587+
yyjson_val *source_obj = yyjson_arr_get(row, 7);
3588+
yyjson_val *source_val = yyjson_obj_get(source_obj, "source");
3589+
ASSERT_NOT_NULL(source_val);
3590+
ASSERT_STR_EQ(yyjson_get_str(source_val), source);
3591+
yyjson_doc_free(doc);
3592+
3593+
free(inner);
3594+
free(resp);
3595+
cbm_mcp_server_free(srv);
3596+
cbm_unlink(source_path);
3597+
cbm_rmdir(design_dir);
3598+
cbm_rmdir(project_dir);
3599+
cbm_rmdir(tmp);
3600+
PASS();
3601+
}
3602+
35333603
/* Reproduce-first (#687): scoped content search over a repo whose ROOT PATH
35343604
* contains a space. write_scoped_filelist emits "<root>/<file>" records that the
35353605
* Unix pipeline pipes to grep via xargs. With plain `xargs` (newline-split) the
@@ -9410,6 +9480,7 @@ SUITE(mcp) {
94109480
RUN_TEST(tool_search_code_missing_pattern);
94119481
RUN_TEST(tool_search_code_no_project);
94129482
RUN_TEST(search_code_multi_word);
9483+
RUN_TEST(search_code_full_preserves_utf8_source);
94139484
RUN_TEST(search_code_scoped_path_with_spaces_issue687);
94149485
#ifdef _WIN32
94159486
RUN_TEST(search_code_scoped_path_with_cjk_root_issue903);

0 commit comments

Comments
 (0)