Skip to content

Commit f9d7f4a

Browse files
committed
feat(index): persist coverage in separate table + queryable miss graph
Persist the parse-coverage signal outside the graph and make the misses queryable both as a list and as a graph. - store: new index_coverage table (project, rel_path, kind, detail) — coverage is metadata ABOUT the graph and never mixes into the graph tables; replace/get/free APIs with a deleted-file prune keyed off file_hashes (the live-file set) - store: materialize a derived miss graph under the shadow project "<project>::coverage" (Project -> Folder chain -> File{kind,detail}, CONTAINS_FOLDER/CONTAINS_FILE edges), rebuilt from the table inside the same transaction on every replace; invisible to list_projects (which scans the cache directory, not the projects table) - pipeline: coverage is written in the same persist step as every other post-dump SQL write (hashes, ADR, FTS). Full runs replace the whole set; incremental runs merge rows surviving from files not re-extracted with this run's fresh entries, so a fixed file's flag clears and deleted files prune automatically - mcp: new get_index_coverage tool (per-project parse_partial + skipped lists with the best-effort note); query_graph gains graph="coverage" to run the same cypher against the miss graph; the File-node property marker from the previous commit is dropped in favor of the separate table — the code graph gains no coverage rows - wording: constructs in flagged ranges MAY be missing — tree-sitter error recovery still salvages some (verified: a function inside a flagged #ifdef region and even a broken def were still extracted) - tests: store round-trip/prune/shadow-graph unit test; e2e asserts the response fields, the advertised coverage-graph query, code-graph purity, and that the flag clears after the file is fixed (incremental route) Refs DeusData#963 Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent a4c17bd commit f9d7f4a

10 files changed

Lines changed: 667 additions & 107 deletions

File tree

‎src/graph_buffer/graph_buffer.c‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -769,24 +769,6 @@ const cbm_gbuf_node_t *cbm_gbuf_find_by_qn(const cbm_gbuf_t *gb, const char *qn)
769769
return cbm_ht_get(gb->node_by_qn, qn);
770770
}
771771

772-
bool cbm_gbuf_set_node_props(cbm_gbuf_t *gb, const char *qualified_name,
773-
const char *properties_json) {
774-
if (!gb || !qualified_name || !properties_json) {
775-
return false;
776-
}
777-
cbm_gbuf_node_t *node = cbm_ht_get(gb->node_by_qn, qualified_name);
778-
if (!node) {
779-
return false;
780-
}
781-
char *new_props = heap_strdup(properties_json);
782-
if (!new_props) {
783-
return false;
784-
}
785-
free(node->properties_json);
786-
node->properties_json = new_props;
787-
return true;
788-
}
789-
790772
const cbm_gbuf_node_t *cbm_gbuf_find_by_id(const cbm_gbuf_t *gb, int64_t id) {
791773
if (!gb || !gb->by_id || id < 0 || id >= gb->by_id_cap) {
792774
return NULL;

‎src/graph_buffer/graph_buffer.h‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -73,13 +73,6 @@ int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name
7373
const char *qualified_name, const char *file_path, int start_line,
7474
int end_line, const char *properties_json);
7575

76-
/* Replace an existing node's properties_json by qualified name (copied).
77-
* Bypasses the upsert survivor rule — use for post-hoc annotation of a node
78-
* that is known to exist (e.g. parse-coverage markers on File nodes, #963).
79-
* Returns true when the node was found and updated. */
80-
bool cbm_gbuf_set_node_props(cbm_gbuf_t *gb, const char *qualified_name,
81-
const char *properties_json);
82-
8376
/* Find a node by qualified name. Returns NULL if not found. */
8477
const cbm_gbuf_node_t *cbm_gbuf_find_by_qn(const cbm_gbuf_t *gb, const char *qn);
8578

‎src/mcp/mcp.c‎

Lines changed: 136 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -319,11 +319,10 @@ static const tool_def_t TOOLS[] = {
319319
"Requires target_projects param. Ensure target projects have fresh indexes first. "
320320
"COVERAGE: the response reports files that were NOT fully indexed — 'skipped' (not "
321321
"indexed at all: oversized/read/parse failures) and 'parse_partial' (indexed, but "
322-
"constructs inside the listed line ranges could not be parsed and are absent from the "
323-
"graph). parse_partial files also carry {parse_incomplete:true, error_ranges} on their "
324-
"File node — query later via query_graph: MATCH (f:File) WHERE f.parse_incomplete = true "
325-
"RETURN f. Both signals are best-effort: absence of a flag is NOT a completeness "
326-
"guarantee; prefer grep inside flagged ranges.",
322+
"constructs inside the listed line ranges could not be parsed and MAY be missing from "
323+
"the graph). Query the persisted signal any time via the get_index_coverage tool or "
324+
"structurally via query_graph(graph=\"coverage\"). Both signals are best-effort: absence "
325+
"of a flag is NOT a completeness guarantee; prefer grep inside flagged ranges.",
327326
"{\"type\":\"object\",\"properties\":{\"repo_path\":{\"type\":\"string\",\"description\":"
328327
"\"Path to the repository\"},"
329328
"\"mode\":{\"type\":\"string\","
@@ -395,9 +394,20 @@ static const tool_def_t TOOLS[] = {
395394
"no conditionally-guarded base case), param_count and max_access_depth (structure smells). "
396395
"Find all hot-path candidates in one query, e.g. MATCH (f:Function) WHERE "
397396
"f.transitive_loop_depth >= 3 OR f.linear_scan_in_loop >= 1 RETURN f.qualified_name, "
398-
"f.transitive_loop_depth, f.linear_scan_in_loop ORDER BY f.transitive_loop_depth DESC.",
397+
"f.transitive_loop_depth, f.linear_scan_in_loop ORDER BY f.transitive_loop_depth DESC. "
398+
"COVERAGE GRAPH: pass graph=\"coverage\" to query the best-effort miss graph instead — "
399+
"the file structure of files the indexer could NOT fully cover (Project → Folder → "
400+
"File nodes with CONTAINS_FOLDER/CONTAINS_FILE edges; each File carries kind "
401+
"(\"parse_partial\" = indexed but constructs in the flagged line ranges MAY be missing; "
402+
"or a skip phase) and detail (the line ranges / reason)). Example: MATCH (f:File) WHERE "
403+
"f.kind = \\\"parse_partial\\\" RETURN f.file_path, f.detail. Absence from this graph is "
404+
"NOT a completeness guarantee.",
399405
"{\"type\":\"object\",\"properties\":{\"query\":{\"type\":\"string\",\"description\":\"Cypher "
400-
"query\"},\"project\":{\"type\":\"string\"},\"max_rows\":{\"type\":\"integer\","
406+
"query\"},\"project\":{\"type\":\"string\"},"
407+
"\"graph\":{\"type\":\"string\",\"enum\":[\"code\",\"coverage\"],\"default\":\"code\","
408+
"\"description\":\"Which graph to query: the code knowledge graph (default) or the "
409+
"indexing-coverage miss graph (files not fully indexed, as a file-structure graph).\"},"
410+
"\"max_rows\":{\"type\":\"integer\","
401411
"\"description\":"
402412
"\"Optional row limit. Default: unlimited up to a 100k row "
403413
"ceiling. No offset support — use search_graph for paginated browsing.\"}},"
@@ -489,6 +499,18 @@ static const tool_def_t TOOLS[] = {
489499
"{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":["
490500
"\"project\"]}"},
491501

502+
{"get_index_coverage", "Index coverage",
503+
"Report which files the indexer could NOT fully cover (best-effort signal, #963): "
504+
"'parse_partial' files WERE indexed but contain line ranges tree-sitter could not parse — "
505+
"constructs there MAY be missing from the graph (some are still recovered); 'skipped' files "
506+
"were not indexed at all (oversized/read/parse failure). Use this before trusting graph "
507+
"completeness on a file: if a file is listed, ALSO grep it (especially the flagged ranges). "
508+
"IMPORTANT: absence from this list is NOT a completeness guarantee — the signal only marks "
509+
"what the indexer can detect. For structural queries over the misses use "
510+
"query_graph(graph=\"coverage\").",
511+
"{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":["
512+
"\"project\"]}"},
513+
492514
{"detect_changes", "Detect changes", "Detect code changes and their impact",
493515
"{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"},\"scope\":{\"type\":"
494516
"\"string\"},\"depth\":{\"type\":\"integer\",\"default\":2},\"base_branch\":{\"type\":"
@@ -2065,10 +2087,21 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) {
20652087
cbm_store_t *store = resolve_store(srv, project);
20662088
int max_rows = cbm_mcp_get_int_arg(args, "max_rows", 0);
20672089

2090+
/* graph="coverage" (#963): run the SAME cypher against the derived
2091+
* miss-graph view (shadow project "<project>::coverage") instead of the
2092+
* code graph — file structure of not-fully-indexed files only. */
2093+
char *graph_arg = cbm_mcp_get_string_arg(args, "graph");
2094+
bool coverage_graph = graph_arg && strcmp(graph_arg, "coverage") == 0;
2095+
free(graph_arg);
2096+
20682097
if (!query) {
20692098
free(project);
20702099
return cbm_mcp_text_result("query is required", true);
20712100
}
2101+
if (coverage_graph && !project) {
2102+
free(query);
2103+
return cbm_mcp_text_result("project is required when graph=\"coverage\"", true);
2104+
}
20722105
if (!store) {
20732106
char *_err = build_project_list_error("project not found or not indexed");
20742107
char *_res = cbm_mcp_text_result(_err, true);
@@ -2085,8 +2118,15 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) {
20852118
return not_indexed;
20862119
}
20872120

2121+
char covproj[CBM_SZ_512];
2122+
const char *cypher_project = project;
2123+
if (coverage_graph) {
2124+
cbm_store_coverage_shadow_project(covproj, sizeof(covproj), project);
2125+
cypher_project = covproj;
2126+
}
2127+
20882128
cbm_cypher_result_t result = {0};
2089-
int rc = cbm_cypher_execute(store, query, project, max_rows, &result);
2129+
int rc = cbm_cypher_execute(store, query, cypher_project, max_rows, &result);
20902130

20912131
if (rc < 0) {
20922132
char *err_msg = result.error ? result.error : "query execution failed";
@@ -2181,6 +2221,87 @@ static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) {
21812221
return result;
21822222
}
21832223

2224+
/* get_index_coverage (#963): report the best-effort indexing-coverage signal
2225+
* from the separate index_coverage table (coverage is metadata ABOUT the
2226+
* graph, stored outside it). Full per-project list, capped generously. */
2227+
enum { COVERAGE_FILE_CAP = 500 };
2228+
2229+
static char *handle_get_index_coverage(cbm_mcp_server_t *srv, const char *args) {
2230+
char *project = get_project_arg(args);
2231+
cbm_store_t *store = resolve_store(srv, project);
2232+
REQUIRE_STORE(store, project);
2233+
2234+
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
2235+
yyjson_mut_val *root = yyjson_mut_obj(doc);
2236+
yyjson_mut_doc_set_root(doc, root);
2237+
2238+
if (project) {
2239+
yyjson_mut_obj_add_str(doc, root, "project", project);
2240+
cbm_coverage_row_t *rows = NULL;
2241+
int count = 0;
2242+
(void)cbm_store_coverage_get(store, project, &rows, &count);
2243+
2244+
yyjson_mut_val *pp_files = yyjson_mut_arr(doc);
2245+
yyjson_mut_val *sk_files = yyjson_mut_arr(doc);
2246+
int pp_n = 0;
2247+
int sk_n = 0;
2248+
for (int i = 0; i < count; i++) {
2249+
bool partial = rows[i].kind && strcmp(rows[i].kind, "parse_partial") == 0;
2250+
if (partial) {
2251+
if (pp_n < COVERAGE_FILE_CAP) {
2252+
yyjson_mut_val *fe = yyjson_mut_obj(doc);
2253+
yyjson_mut_obj_add_strcpy(doc, fe, "path", rows[i].rel_path);
2254+
yyjson_mut_obj_add_strcpy(doc, fe, "error_ranges",
2255+
rows[i].detail ? rows[i].detail : "");
2256+
yyjson_mut_arr_add_val(pp_files, fe);
2257+
}
2258+
pp_n++;
2259+
} else {
2260+
if (sk_n < COVERAGE_FILE_CAP) {
2261+
yyjson_mut_val *fe = yyjson_mut_obj(doc);
2262+
yyjson_mut_obj_add_strcpy(doc, fe, "path", rows[i].rel_path);
2263+
yyjson_mut_obj_add_strcpy(doc, fe, "reason",
2264+
rows[i].detail ? rows[i].detail : "");
2265+
yyjson_mut_obj_add_strcpy(doc, fe, "phase", rows[i].kind ? rows[i].kind : "");
2266+
yyjson_mut_arr_add_val(sk_files, fe);
2267+
}
2268+
sk_n++;
2269+
}
2270+
}
2271+
cbm_store_free_coverage(rows, count);
2272+
2273+
yyjson_mut_val *pp = yyjson_mut_obj(doc);
2274+
yyjson_mut_obj_add_val(doc, pp, "files", pp_files);
2275+
yyjson_mut_obj_add_int(doc, pp, "count", pp_n);
2276+
yyjson_mut_obj_add_bool(doc, pp, "truncated", pp_n > COVERAGE_FILE_CAP);
2277+
yyjson_mut_obj_add_val(doc, root, "parse_partial", pp);
2278+
2279+
yyjson_mut_val *sk = yyjson_mut_obj(doc);
2280+
yyjson_mut_obj_add_val(doc, sk, "files", sk_files);
2281+
yyjson_mut_obj_add_int(doc, sk, "count", sk_n);
2282+
yyjson_mut_obj_add_bool(doc, sk, "truncated", sk_n > COVERAGE_FILE_CAP);
2283+
yyjson_mut_obj_add_val(doc, root, "skipped", sk);
2284+
2285+
yyjson_mut_obj_add_str(
2286+
doc, root, "note",
2287+
"Best-effort signal, not a completeness guarantee: parse_partial files WERE indexed, "
2288+
"but constructs inside the listed line ranges (1-based) MAY be missing from the graph "
2289+
"(tree-sitter error recovery still salvages some). skipped files were not indexed at "
2290+
"all. Prefer text search (grep) for flagged files/ranges. Files absent from this list "
2291+
"are NOT guaranteed to be fully indexed.");
2292+
} else {
2293+
yyjson_mut_obj_add_str(doc, root, "status", "no_project");
2294+
}
2295+
2296+
char *json = yy_doc_to_str(doc);
2297+
yyjson_mut_doc_free(doc);
2298+
free(project);
2299+
2300+
char *result = cbm_mcp_text_result(json, false);
2301+
free(json);
2302+
return result;
2303+
}
2304+
21842305
/* delete_project: just erase the .db file (and WAL/SHM). */
21852306
static char *handle_delete_project(cbm_mcp_server_t *srv, const char *args) {
21862307
char *name = get_project_arg(args);
@@ -3396,9 +3517,10 @@ static void add_parse_partial_summary(yyjson_mut_doc *doc, yyjson_mut_val *root,
33963517
yyjson_mut_obj_add_str(doc, pp, "note",
33973518
"Best-effort signal, not a completeness guarantee: these files WERE "
33983519
"indexed, but constructs inside the listed line ranges (1-based) could "
3399-
"not be parsed and are missing from the graph. Prefer text search "
3400-
"(grep) for those regions. Files absent from this list are NOT "
3401-
"guaranteed to be fully indexed.");
3520+
"not be parsed and MAY be missing from the graph (tree-sitter error "
3521+
"recovery still salvages some). Prefer text search (grep) for those "
3522+
"regions. Files absent from this list are NOT guaranteed to be fully "
3523+
"indexed. Query the persisted signal via get_index_coverage.");
34023524
yyjson_mut_obj_add_val(doc, root, "parse_partial", pp);
34033525
}
34043526

@@ -5856,6 +5978,9 @@ char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const ch
58565978
if (strcmp(tool_name, "query_graph") == 0) {
58575979
return handle_query_graph(srv, args_json);
58585980
}
5981+
if (strcmp(tool_name, "get_index_coverage") == 0) {
5982+
return handle_get_index_coverage(srv, args_json);
5983+
}
58595984
if (strcmp(tool_name, "index_status") == 0) {
58605985
return handle_index_status(srv, args_json);
58615986
}

‎src/pipeline/pipeline.c‎

Lines changed: 23 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -315,40 +315,6 @@ void cbm_pipeline_get_file_errors(const cbm_pipeline_t *p, cbm_file_error_t **ou
315315
}
316316
}
317317

318-
void cbm_pipeline_stamp_parse_partial(cbm_pipeline_t *p, cbm_gbuf_t *gbuf) {
319-
if (!p || !gbuf) {
320-
return;
321-
}
322-
int stamped = 0;
323-
for (int i = 0; i < p->file_errors_count; i++) {
324-
const cbm_file_error_t *e = &p->file_errors[i];
325-
if (!e->path || !e->phase || strcmp(e->phase, "parse_partial") != 0) {
326-
continue;
327-
}
328-
char *file_qn = cbm_pipeline_fqn_compute(p->project_name, e->path, "__file__");
329-
if (!file_qn) {
330-
continue;
331-
}
332-
const char *slash = strrchr(e->path, '/');
333-
const char *basename = slash ? slash + SKIP_ONE : e->path;
334-
const char *ext = strrchr(basename, '.');
335-
/* Full replacement props (set_node_props does not merge): keep the
336-
* "extension" key pass_structure wrote, add the coverage marker. The
337-
* ranges string is digits/commas/dashes — JSON-safe unescaped. */
338-
char props[CBM_SZ_2K];
339-
snprintf(props, sizeof(props),
340-
"{\"extension\":\"%s\",\"parse_incomplete\":true,\"error_ranges\":\"%s\"}",
341-
ext ? ext : "", e->reason ? e->reason : "");
342-
if (cbm_gbuf_set_node_props(gbuf, file_qn, props)) {
343-
stamped++;
344-
}
345-
free(file_qn);
346-
}
347-
if (stamped > 0) {
348-
cbm_log_info("index.parse_partial", "files", itoa_buf(stamped));
349-
}
350-
}
351-
352318
void cbm_pipeline_get_committed_counts(const cbm_pipeline_t *p, int *nodes, int *edges) {
353319
if (nodes) {
354320
*nodes = p ? p->committed_nodes : -1;
@@ -1148,6 +1114,29 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *fil
11481114
}
11491115
CBM_PROF_END_N("persist", "4_file_hashes", t_fh, file_count);
11501116

1117+
/* Coverage rows (#963): a full run's file_errors is the complete
1118+
* coverage truth for the project. The dump recreated the DB file, so
1119+
* the separate index_coverage table starts empty — write only when
1120+
* there is something to record (AFTER hashes, so the deleted-file
1121+
* prune inside replace sees the live file set). */
1122+
if (p->file_errors_count > 0) {
1123+
cbm_coverage_row_t *cov =
1124+
(cbm_coverage_row_t *)malloc((size_t)p->file_errors_count * sizeof(*cov));
1125+
if (cov) {
1126+
for (int i = 0; i < p->file_errors_count; i++) {
1127+
cov[i].rel_path = p->file_errors[i].path;
1128+
cov[i].kind = p->file_errors[i].phase;
1129+
cov[i].detail = p->file_errors[i].reason;
1130+
}
1131+
if (cbm_store_coverage_replace(hash_store, p->project_name, cov,
1132+
p->file_errors_count) != CBM_STORE_OK) {
1133+
cbm_log_error("pipeline.err", "phase", "persist_coverage", "project",
1134+
p->project_name);
1135+
}
1136+
free(cov);
1137+
}
1138+
}
1139+
11511140
/* FTS5 backfill: populate nodes_fts with camelCase-split names.
11521141
* Contentless FTS5 requires the special 'delete-all' command instead of
11531142
* DELETE FROM to wipe prior rows (there's no underlying content table).
@@ -1296,11 +1285,6 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
12961285
if (check_cancel(p)) {
12971286
return CBM_NOT_FOUND;
12981287
}
1299-
if (rc == 0) {
1300-
/* Extraction merged its per-file errors — stamp parse-partial File
1301-
* nodes (#963) before downstream passes and the dump. */
1302-
cbm_pipeline_stamp_parse_partial(p, p->gbuf);
1303-
}
13041288
return rc;
13051289
}
13061290

0 commit comments

Comments
 (0)