Skip to content

Commit 9a547b3

Browse files
authored
Merge pull request DeusData#1253 from anfedoro/agent/fix-search-code-utf8
fix: preserve UTF-8 in search_code source
2 parents 78d1b2f + ed14915 commit 9a547b3

2 files changed

Lines changed: 273 additions & 22 deletions

File tree

‎src/mcp/mcp.c‎

Lines changed: 25 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8766,16 +8766,6 @@ static char *handle_get_code_snippet(cbm_mcp_server_t *srv, const char *args) {
87668766

87678767
/* ── search_code v2: graph-augmented code search ─────────────── */
87688768

8769-
/* Strip non-ASCII bytes to guarantee valid UTF-8 JSON output */
8770-
enum { ASCII_MAX = 127 };
8771-
static void sanitize_ascii(char *s) {
8772-
for (unsigned char *p = (unsigned char *)s; *p; p++) {
8773-
if (*p > ASCII_MAX) {
8774-
*p = '?';
8775-
}
8776-
}
8777-
}
8778-
87798769
/* Intermediate grep match */
87808770
typedef struct {
87818771
char file[CBM_SZ_512];
@@ -9016,8 +9006,11 @@ static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, sear
90169006
}
90179007
char *source = read_file_lines(abs_path, s, e);
90189008
if (source) {
9019-
sanitize_ascii(source);
9020-
yyjson_mut_obj_add_strcpy(doc, item, "source", source);
9009+
char *safe_source = sanitize_utf8_lossy(source);
9010+
if (safe_source) {
9011+
yyjson_mut_obj_add_strcpy(doc, item, "source", safe_source);
9012+
free(safe_source);
9013+
}
90219014
free(source);
90229015
if (truncated) {
90239016
yyjson_mut_obj_add_int(doc, item, "source_start", s);
@@ -9032,8 +9025,11 @@ static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, sear
90329025
}
90339026
char *ctx = read_file_lines(abs_path, ctx_start, ctx_end);
90349027
if (ctx) {
9035-
sanitize_ascii(ctx);
9036-
yyjson_mut_obj_add_strcpy(doc, item, "context", ctx);
9028+
char *safe_context = sanitize_utf8_lossy(ctx);
9029+
if (safe_context) {
9030+
yyjson_mut_obj_add_strcpy(doc, item, "context", safe_context);
9031+
free(safe_context);
9032+
}
90379033
yyjson_mut_obj_add_int(doc, item, "context_start", ctx_start);
90389034
free(ctx);
90399035
}
@@ -9205,16 +9201,17 @@ static char *assemble_search_output(search_result_t *sr, int sr_count, grep_matc
92059201
build_dedup_files_array(doc, sr, output_count, raw, raw_count));
92069202
} else {
92079203
/* json-stringified tree: cols + column-ordered row arrays. FULL mode
9208-
* appends a per-row object cell with the (guarded, windowed) source —
9209-
* attach_result_source semantics unchanged. */
9204+
* appends a per-row object cell with the (guarded, windowed) source;
9205+
* context requests append the corresponding context object. */
9206+
bool attach_context = context_lines > 0 && mode != MODE_FULL;
92109207
yyjson_mut_val *jcols = yyjson_mut_arr(doc);
92119208
static const char *const sc_cols[] = {"qn", "label", "file", "lines",
92129209
"matches", "in", "out"};
92139210
for (size_t ci = 0; ci < sizeof(sc_cols) / sizeof(sc_cols[0]); ci++) {
92149211
yyjson_mut_arr_add_str(doc, jcols, sc_cols[ci]);
92159212
}
9216-
if (mode == MODE_FULL) {
9217-
yyjson_mut_arr_add_str(doc, jcols, "source");
9213+
if (mode == MODE_FULL || attach_context) {
9214+
yyjson_mut_arr_add_str(doc, jcols, mode == MODE_FULL ? "source" : "context");
92189215
}
92199216
yyjson_mut_obj_add_val(doc, root_obj, "cols", jcols);
92209217

@@ -9240,7 +9237,7 @@ static char *assemble_search_output(search_result_t *sr, int sr_count, grep_matc
92409237
yyjson_mut_arr_add_val(row, ml);
92419238
yyjson_mut_arr_add_int(doc, row, r->in_degree);
92429239
yyjson_mut_arr_add_int(doc, row, r->out_degree);
9243-
if (mode == MODE_FULL) {
9240+
if (mode == MODE_FULL || attach_context) {
92449241
yyjson_mut_val *src = yyjson_mut_obj(doc);
92459242
attach_result_source(doc, src, r, mode, context_lines, root_path);
92469243
yyjson_mut_arr_add_val(row, src);
@@ -9312,7 +9309,11 @@ static char *assemble_search_output(search_result_t *sr, int sr_count, grep_matc
93129309

93139310
char *json = yy_doc_to_str(doc);
93149311
if (json) {
9315-
sanitize_ascii(json);
9312+
char *safe_json = sanitize_utf8_lossy(json);
9313+
if (safe_json) {
9314+
free(json);
9315+
json = safe_json;
9316+
}
93169317
}
93179318
yyjson_mut_doc_free(doc);
93189319

@@ -9383,8 +9384,10 @@ static grep_match_t *collect_grep_matches(FILE *fp, const char *root_path, size_
93839384
safe_grow(gm, gm_count, gm_cap, PAIR_LEN);
93849385
snprintf(gm[gm_count].file, sizeof(gm[0].file), "%s", file);
93859386
gm[gm_count].line = (int)strtol(sep1 + SKIP_ONE, NULL, CBM_DECIMAL_BASE);
9386-
snprintf(gm[gm_count].content, sizeof(gm[0].content), "%s", sep2 + SKIP_ONE);
9387-
sanitize_ascii(gm[gm_count].content);
9387+
char *safe_content = sanitize_utf8_lossy(sep2 + SKIP_ONE);
9388+
snprintf(gm[gm_count].content, sizeof(gm[0].content), "%s",
9389+
safe_content ? safe_content : sep2 + SKIP_ONE);
9390+
free(safe_content);
93889391
gm_count++;
93899392
}
93909393

‎tests/test_mcp.c‎

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4311,6 +4311,250 @@ TEST(search_code_multi_word) {
43114311
PASS();
43124312
}
43134313

4314+
/* Regression guard: search_code full results must preserve valid UTF-8 source. */
4315+
static bool is_valid_json_response(const char *json);
4316+
4317+
TEST(search_code_full_preserves_utf8_source) {
4318+
char tmp[512];
4319+
snprintf(tmp, sizeof(tmp), "/tmp/cbm_srch_utf8_XXXXXX");
4320+
ASSERT_TRUE(cbm_mkdtemp(tmp) != NULL);
4321+
4322+
char project_dir[640];
4323+
snprintf(project_dir, sizeof(project_dir), "%s/project", tmp);
4324+
ASSERT_EQ(cbm_mkdir(project_dir), 0);
4325+
char design_dir[768];
4326+
snprintf(design_dir, sizeof(design_dir), "%s/design", project_dir);
4327+
ASSERT_EQ(cbm_mkdir(design_dir), 0);
4328+
4329+
char source_path[768];
4330+
snprintf(source_path, sizeof(source_path), "%s/design.md", design_dir);
4331+
FILE *fp = cbm_fopen(source_path, "wb");
4332+
ASSERT_NOT_NULL(fp);
4333+
const char source[] = "# accounting-design\nРусский текст: бухгалтерский учет.\n";
4334+
ASSERT_EQ(fwrite(source, 1, sizeof(source) - SKIP_ONE, fp), sizeof(source) - SKIP_ONE);
4335+
ASSERT_EQ(fclose(fp), 0);
4336+
4337+
cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);
4338+
ASSERT_NOT_NULL(srv);
4339+
cbm_store_t *st = cbm_mcp_server_store(srv);
4340+
ASSERT_NOT_NULL(st);
4341+
const char *project = "utf8-search";
4342+
cbm_mcp_server_set_project(srv, project);
4343+
cbm_store_upsert_project(st, project, project_dir);
4344+
4345+
cbm_node_t section = {.project = project,
4346+
.label = "Section",
4347+
.name = "accounting-design",
4348+
.qualified_name = "utf8-search.design.accounting-design",
4349+
.file_path = "design/design.md",
4350+
.start_line = 1,
4351+
.end_line = 2};
4352+
ASSERT_GT(cbm_store_upsert_node(st, &section), 0);
4353+
4354+
char *resp = cbm_mcp_server_handle(
4355+
srv, "{\"jsonrpc\":\"2.0\",\"id\":97,\"method\":\"tools/call\","
4356+
"\"params\":{\"name\":\"search_code\",\"arguments\":{"
4357+
"\"project\":\"utf8-search\",\"pattern\":\"accounting-design\","
4358+
"\"file_pattern\":\"*.md\",\"path_filter\":\"^design/\","
4359+
"\"mode\":\"full\",\"format\":\"json\",\"limit\":5}}}");
4360+
ASSERT_NOT_NULL(resp);
4361+
char *inner = extract_text_content(resp);
4362+
ASSERT_NOT_NULL(inner);
4363+
4364+
yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0);
4365+
ASSERT_NOT_NULL(doc);
4366+
yyjson_val *rows = yyjson_obj_get(yyjson_doc_get_root(doc), "rows");
4367+
ASSERT_NOT_NULL(rows);
4368+
ASSERT_TRUE(yyjson_arr_size(rows) > 0);
4369+
yyjson_val *row = yyjson_arr_get(rows, 0);
4370+
yyjson_val *source_obj = yyjson_arr_get(row, 7);
4371+
yyjson_val *source_val = yyjson_obj_get(source_obj, "source");
4372+
ASSERT_NOT_NULL(source_val);
4373+
ASSERT_STR_EQ(yyjson_get_str(source_val), source);
4374+
yyjson_doc_free(doc);
4375+
4376+
free(inner);
4377+
free(resp);
4378+
cbm_mcp_server_free(srv);
4379+
cbm_unlink(source_path);
4380+
cbm_rmdir(design_dir);
4381+
cbm_rmdir(project_dir);
4382+
cbm_rmdir(tmp);
4383+
PASS();
4384+
}
4385+
4386+
TEST(search_code_raw_match_preserves_utf8_content) {
4387+
char tmp[256];
4388+
cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp));
4389+
ASSERT_NOT_NULL(srv);
4390+
4391+
char raw_path[512];
4392+
snprintf(raw_path, sizeof(raw_path), "%s/project/raw.md", tmp);
4393+
const char raw_source[] = "header\nraw-Русский content\n";
4394+
FILE *fp = cbm_fopen(raw_path, "wb");
4395+
ASSERT_NOT_NULL(fp);
4396+
ASSERT_EQ(fwrite(raw_source, 1, sizeof(raw_source) - SKIP_ONE, fp),
4397+
sizeof(raw_source) - SKIP_ONE);
4398+
ASSERT_EQ(fclose(fp), 0);
4399+
4400+
cbm_store_t *st = cbm_mcp_server_store(srv);
4401+
ASSERT_NOT_NULL(st);
4402+
cbm_node_t node = {.project = "test-project",
4403+
.label = "Section",
4404+
.name = "raw",
4405+
.qualified_name = "test-project.raw",
4406+
.file_path = "raw.md",
4407+
.start_line = 1,
4408+
.end_line = 1};
4409+
ASSERT_GT(cbm_store_upsert_node(st, &node), 0);
4410+
4411+
char *resp = cbm_mcp_server_handle(
4412+
srv, "{\"jsonrpc\":\"2.0\",\"id\":98,\"method\":\"tools/call\","
4413+
"\"params\":{\"name\":\"search_code\",\"arguments\":{"
4414+
"\"project\":\"test-project\",\"pattern\":\"raw-\","
4415+
"\"file_pattern\":\"*.md\",\"format\":\"json\",\"limit\":5}}}");
4416+
ASSERT_NOT_NULL(resp);
4417+
char *inner = extract_text_content(resp);
4418+
ASSERT_NOT_NULL(inner);
4419+
yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0);
4420+
ASSERT_NOT_NULL(doc);
4421+
yyjson_val *raw_obj = yyjson_obj_get(yyjson_doc_get_root(doc), "raw_matches");
4422+
ASSERT_NOT_NULL(raw_obj);
4423+
yyjson_val *raw_rows = yyjson_obj_get(raw_obj, "rows");
4424+
ASSERT_NOT_NULL(raw_rows);
4425+
ASSERT_TRUE(yyjson_arr_size(raw_rows) > 0);
4426+
yyjson_val *raw_row = yyjson_arr_get(raw_rows, 0);
4427+
ASSERT_STR_EQ(yyjson_get_str(yyjson_arr_get(raw_row, 2)), "raw-Русский content");
4428+
yyjson_doc_free(doc);
4429+
4430+
free(inner);
4431+
free(resp);
4432+
cbm_mcp_server_free(srv);
4433+
cleanup_snippet_dir(tmp);
4434+
PASS();
4435+
}
4436+
4437+
TEST(search_code_context_preserves_utf8_context) {
4438+
char tmp[256];
4439+
cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp));
4440+
ASSERT_NOT_NULL(srv);
4441+
4442+
char context_path[512];
4443+
snprintf(context_path, sizeof(context_path), "%s/project/context.md", tmp);
4444+
const char context_source[] = "before-до\ncontext-needle\nпосле-после\n";
4445+
FILE *fp = cbm_fopen(context_path, "wb");
4446+
ASSERT_NOT_NULL(fp);
4447+
ASSERT_EQ(fwrite(context_source, 1, sizeof(context_source) - SKIP_ONE, fp),
4448+
sizeof(context_source) - SKIP_ONE);
4449+
ASSERT_EQ(fclose(fp), 0);
4450+
4451+
cbm_store_t *st = cbm_mcp_server_store(srv);
4452+
ASSERT_NOT_NULL(st);
4453+
cbm_node_t node = {.project = "test-project",
4454+
.label = "Section",
4455+
.name = "context",
4456+
.qualified_name = "test-project.context",
4457+
.file_path = "context.md",
4458+
.start_line = 1,
4459+
.end_line = 3};
4460+
ASSERT_GT(cbm_store_upsert_node(st, &node), 0);
4461+
4462+
char *resp = cbm_mcp_server_handle(
4463+
srv, "{\"jsonrpc\":\"2.0\",\"id\":99,\"method\":\"tools/call\","
4464+
"\"params\":{\"name\":\"search_code\",\"arguments\":{"
4465+
"\"project\":\"test-project\",\"pattern\":\"context-needle\","
4466+
"\"format\":\"json\",\"context\":1,\"limit\":5}}}");
4467+
ASSERT_NOT_NULL(resp);
4468+
char *inner = extract_text_content(resp);
4469+
ASSERT_NOT_NULL(inner);
4470+
yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0);
4471+
ASSERT_NOT_NULL(doc);
4472+
yyjson_val *root = yyjson_doc_get_root(doc);
4473+
yyjson_val *cols = yyjson_obj_get(root, "cols");
4474+
yyjson_val *rows = yyjson_obj_get(root, "rows");
4475+
ASSERT_NOT_NULL(cols);
4476+
ASSERT_NOT_NULL(rows);
4477+
ASSERT_TRUE(yyjson_arr_size(rows) > 0);
4478+
4479+
size_t context_index = SIZE_MAX;
4480+
size_t col_count = yyjson_arr_size(cols);
4481+
for (size_t i = 0; i < col_count; i++) {
4482+
const char *col = yyjson_get_str(yyjson_arr_get(cols, i));
4483+
if (col && strcmp(col, "context") == 0) {
4484+
context_index = i;
4485+
break;
4486+
}
4487+
}
4488+
ASSERT_TRUE(context_index != SIZE_MAX);
4489+
yyjson_val *context_obj = yyjson_arr_get(yyjson_arr_get(rows, 0), context_index);
4490+
ASSERT_NOT_NULL(context_obj);
4491+
ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(context_obj, "context")), context_source);
4492+
ASSERT_EQ(yyjson_get_int(yyjson_obj_get(context_obj, "context_start")), 1);
4493+
yyjson_doc_free(doc);
4494+
4495+
free(inner);
4496+
free(resp);
4497+
cbm_mcp_server_free(srv);
4498+
cleanup_snippet_dir(tmp);
4499+
PASS();
4500+
}
4501+
4502+
TEST(search_code_invalid_utf8_still_returns_valid_json) {
4503+
char tmp[256];
4504+
cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp));
4505+
ASSERT_NOT_NULL(srv);
4506+
4507+
char invalid_path[512];
4508+
snprintf(invalid_path, sizeof(invalid_path), "%s/project/invalid.md", tmp);
4509+
static const unsigned char invalid_source[] = {
4510+
'i', 'n', 'v', 'a', 'l', 'i', 'd', '-', 'n', 'e', 'e', 'd', 'l', 'e', '\n',
4511+
'c', 'o', 'n', 't', 'e', 'x', 't', ' ', 0xFF, '\n',
4512+
};
4513+
FILE *fp = cbm_fopen(invalid_path, "wb");
4514+
ASSERT_NOT_NULL(fp);
4515+
ASSERT_EQ(fwrite(invalid_source, 1, sizeof(invalid_source), fp), sizeof(invalid_source));
4516+
ASSERT_EQ(fclose(fp), 0);
4517+
4518+
cbm_store_t *st = cbm_mcp_server_store(srv);
4519+
ASSERT_NOT_NULL(st);
4520+
cbm_node_t node = {.project = "test-project",
4521+
.label = "Section",
4522+
.name = "invalid",
4523+
.qualified_name = "test-project.invalid",
4524+
.file_path = "invalid.md",
4525+
.start_line = 1,
4526+
.end_line = 2};
4527+
ASSERT_GT(cbm_store_upsert_node(st, &node), 0);
4528+
4529+
char *resp = cbm_mcp_server_handle(
4530+
srv, "{\"jsonrpc\":\"2.0\",\"id\":100,\"method\":\"tools/call\","
4531+
"\"params\":{\"name\":\"search_code\",\"arguments\":{"
4532+
"\"project\":\"test-project\",\"pattern\":\"invalid-needle\","
4533+
"\"mode\":\"full\",\"format\":\"json\",\"limit\":5}}}");
4534+
ASSERT_NOT_NULL(resp);
4535+
char *inner = extract_text_content(resp);
4536+
ASSERT_NOT_NULL(inner);
4537+
ASSERT_TRUE(is_valid_json_response(inner));
4538+
yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0);
4539+
ASSERT_NOT_NULL(doc);
4540+
yyjson_val *rows = yyjson_obj_get(yyjson_doc_get_root(doc), "rows");
4541+
ASSERT_NOT_NULL(rows);
4542+
ASSERT_TRUE(yyjson_arr_size(rows) > 0);
4543+
yyjson_val *source_obj = yyjson_arr_get(yyjson_arr_get(rows, 0), 7);
4544+
ASSERT_NOT_NULL(source_obj);
4545+
const char *safe_source = yyjson_get_str(yyjson_obj_get(source_obj, "source"));
4546+
ASSERT_NOT_NULL(safe_source);
4547+
ASSERT_NOT_NULL(strstr(safe_source, "\xEF\xBF\xBD"));
4548+
ASSERT_NULL(memchr(safe_source, 0xFF, strlen(safe_source)));
4549+
yyjson_doc_free(doc);
4550+
4551+
free(inner);
4552+
free(resp);
4553+
cbm_mcp_server_free(srv);
4554+
cleanup_snippet_dir(tmp);
4555+
PASS();
4556+
}
4557+
43144558
/* Reproduce-first (#687): scoped content search over a repo whose ROOT PATH
43154559
* contains a space. write_scoped_filelist emits "<root>/<file>" records that the
43164560
* Unix pipeline pipes to grep via xargs. With plain `xargs` (newline-split) the
@@ -10967,6 +11211,10 @@ SUITE(mcp) {
1096711211
RUN_TEST(tool_search_code_limit_declares_a_minimum_issue1511);
1096811212
RUN_TEST(tool_search_code_no_project);
1096911213
RUN_TEST(search_code_multi_word);
11214+
RUN_TEST(search_code_full_preserves_utf8_source);
11215+
RUN_TEST(search_code_raw_match_preserves_utf8_content);
11216+
RUN_TEST(search_code_context_preserves_utf8_context);
11217+
RUN_TEST(search_code_invalid_utf8_still_returns_valid_json);
1097011218
RUN_TEST(search_code_scoped_path_with_spaces_issue687);
1097111219
#ifdef _WIN32
1097211220
RUN_TEST(search_code_scoped_path_with_cjk_root_issue903);

0 commit comments

Comments
 (0)