Skip to content

Commit a4c17bd

Browse files
committed
feat(index): best-effort parse-coverage signal for partially parsed files
Files whose parse tree contains tree-sitter ERROR/MISSING regions were silently indexed as if complete — constructs inside those regions are absent from the graph with no signal anywhere (classic trigger: the preprocessor-blind #ifdef-split-brace pattern in C). - cbm_extract_file: set parse_incomplete and record the 1-based line ranges of the top-most error regions (bounded at 64), serialized as "start-end,start-end" into the result arena - pipeline: record such files under the new phase "parse_partial" (distinct from skips — the file IS indexed) in both the parallel and sequential extraction paths; stamp the File node with {parse_incomplete:true, error_ranges} on full and incremental runs via a new cbm_gbuf_set_node_props (upsert survivor rules bypass) - mcp: report parse_partial {files, count, truncated, note} separately from skipped[] (skipped_count semantics unchanged); the per-run logfile lists parse_partial rows; the index_repository tool description documents the signal and the query_graph pattern to find flagged files - tests: new parse_coverage suite (RED-first, 7 cases incl. false- positive guards and the region cap) plus an end-to-end resilience guard asserting response fields, logfile, File-node marker, and the advertised query_graph query The signal is explicitly best-effort: a flag means constructs in the listed ranges were dropped (prefer grep there); the absence of a flag is NOT a completeness guarantee. Refs DeusData#963 Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent ee68144 commit a4c17bd

15 files changed

Lines changed: 627 additions & 17 deletions

‎Makefile.cbm‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -317,6 +317,7 @@ TEST_EXTRACTION_SRCS = \
317317
tests/test_extraction.c \
318318
tests/test_extraction_inheritance.c \
319319
tests/test_extraction_imports.c \
320+
tests/test_parse_coverage.c \
320321
tests/test_grammar_regression.c \
321322
tests/test_grammar_labels.c \
322323
tests/test_grammar_imports.c \

‎internal/cbm/cbm.c‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -688,6 +688,61 @@ static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len,
688688
const char *rel_path, int64_t timeout_micros,
689689
const char **extra_defines, const char **include_paths);
690690

691+
/* Best-effort parse-coverage collection (#963). Walks only the has_error paths
692+
* of the tree and records the 1-based line ranges of the TOP-MOST ERROR/MISSING
693+
* nodes (does not descend into an error subtree — one range per failed region).
694+
* Bounded by CBM_MAX_ERROR_REGIONS so pathological input can't blow up the
695+
* output. The ranges mark where constructs were dropped; they are a detection
696+
* aid, never a completeness proof. */
697+
#define CBM_MAX_ERROR_REGIONS 64
698+
typedef struct {
699+
uint32_t starts[CBM_MAX_ERROR_REGIONS];
700+
uint32_t ends[CBM_MAX_ERROR_REGIONS];
701+
int count;
702+
} cbm_error_regions_t;
703+
704+
static void cbm_error_regions_push(cbm_error_regions_t *acc, TSNode n) {
705+
if (acc->count >= CBM_MAX_ERROR_REGIONS) {
706+
return;
707+
}
708+
acc->starts[acc->count] = ts_node_start_point(n).row + 1;
709+
acc->ends[acc->count] = ts_node_end_point(n).row + 1;
710+
acc->count++;
711+
}
712+
713+
static void cbm_collect_error_regions(TSNode n, cbm_error_regions_t *acc) {
714+
if (acc->count >= CBM_MAX_ERROR_REGIONS) {
715+
return;
716+
}
717+
uint32_t k = ts_node_child_count(n);
718+
for (uint32_t i = 0; i < k && acc->count < CBM_MAX_ERROR_REGIONS; i++) {
719+
TSNode c = ts_node_child(n, i);
720+
if (ts_node_is_missing(c) || strcmp(ts_node_type(c), "ERROR") == 0) {
721+
cbm_error_regions_push(acc, c); /* top-most region; do not descend */
722+
} else if (ts_node_has_error(c)) {
723+
cbm_collect_error_regions(c, acc);
724+
}
725+
}
726+
}
727+
728+
/* Serialize collected regions as "start-end,start-end,..." into the arena. */
729+
static const char *cbm_error_ranges_str(CBMArena *a, const cbm_error_regions_t *regs) {
730+
if (regs->count <= 0) {
731+
return NULL;
732+
}
733+
enum { RANGE_MAX = 24 }; /* "4294967295-4294967295," */
734+
char *buf = (char *)cbm_arena_alloc(a, (size_t)regs->count * RANGE_MAX);
735+
if (!buf) {
736+
return NULL;
737+
}
738+
size_t off = 0;
739+
for (int i = 0; i < regs->count; i++) {
740+
off += (size_t)snprintf(buf + off, RANGE_MAX, "%s%u-%u", i ? "," : "", regs->starts[i],
741+
regs->ends[i]);
742+
}
743+
return buf;
744+
}
745+
691746
/* Public entry: run the extraction and journal completion. The DONE mark on
692747
* every ordinary return (including error/timeout results) tells the crash
693748
* supervisor this file did NOT kill the worker — only a file whose S has no
@@ -788,6 +843,22 @@ static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len,
788843

789844
TSNode root = ts_tree_root_node(tree);
790845

846+
/* Best-effort parse-coverage signal (#963): flag files whose tree contains
847+
* ERROR/MISSING nodes — constructs inside those regions are silently absent
848+
* from the graph. Detection aid only: the absence of this flag is NOT a
849+
* completeness guarantee. */
850+
if (ts_node_has_error(root)) {
851+
result->parse_incomplete = true;
852+
cbm_error_regions_t regs = {{0}, {0}, 0};
853+
if (strcmp(ts_node_type(root), "ERROR") == 0) {
854+
cbm_error_regions_push(&regs, root); /* whole file unparseable */
855+
} else {
856+
cbm_collect_error_regions(root, &regs);
857+
}
858+
result->error_region_count = regs.count;
859+
result->error_ranges = cbm_error_ranges_str(a, &regs);
860+
}
861+
791862
// Compute module QN. Java/Go derive the module from the CONTAINING
792863
// DIRECTORY (package semantics) rather than baking the filename stem in,
793864
// so def QNs, the LSP caller_qn, and the textual calls-enclosing QN all

‎internal/cbm/cbm.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -447,6 +447,16 @@ typedef struct {
447447

448448
bool has_error;
449449
const char *error_msg;
450+
/* Best-effort parse-coverage signal (experimental). parse_incomplete is true
451+
* when the parse tree contains tree-sitter ERROR/MISSING nodes — constructs
452+
* in those regions are silently absent from the graph. error_ranges is a
453+
* compact "start-end,start-end" list of 1-based line ranges (arena-owned) or
454+
* NULL. This only marks what we can DETECT: the absence of a flag is NOT a
455+
* completeness guarantee. Callers should treat a flagged file as "prefer
456+
* grep here", never treat an unflagged file as provably complete. */
457+
bool parse_incomplete;
458+
const char *error_ranges;
459+
int error_region_count;
450460
bool is_test_file;
451461
int imports_count;
452462
TSTree *cached_tree; // retained parse tree (caller frees via cbm_free_tree)

‎src/graph_buffer/graph_buffer.c‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -769,6 +769,24 @@ const cbm_gbuf_node_t *cbm_gbuf_find_by_qn(const cbm_gbuf_t *gb, const char *qn)
769769
return cbm_ht_get(gb->node_by_qn, qn);
770770
}
771771

772+
bool cbm_gbuf_set_node_props(cbm_gbuf_t *gb, const char *qualified_name,
773+
const char *properties_json) {
774+
if (!gb || !qualified_name || !properties_json) {
775+
return false;
776+
}
777+
cbm_gbuf_node_t *node = cbm_ht_get(gb->node_by_qn, qualified_name);
778+
if (!node) {
779+
return false;
780+
}
781+
char *new_props = heap_strdup(properties_json);
782+
if (!new_props) {
783+
return false;
784+
}
785+
free(node->properties_json);
786+
node->properties_json = new_props;
787+
return true;
788+
}
789+
772790
const cbm_gbuf_node_t *cbm_gbuf_find_by_id(const cbm_gbuf_t *gb, int64_t id) {
773791
if (!gb || !gb->by_id || id < 0 || id >= gb->by_id_cap) {
774792
return NULL;

‎src/graph_buffer/graph_buffer.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,13 @@ int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name
7373
const char *qualified_name, const char *file_path, int start_line,
7474
int end_line, const char *properties_json);
7575

76+
/* Replace an existing node's properties_json by qualified name (copied).
77+
* Bypasses the upsert survivor rule — use for post-hoc annotation of a node
78+
* that is known to exist (e.g. parse-coverage markers on File nodes, #963).
79+
* Returns true when the node was found and updated. */
80+
bool cbm_gbuf_set_node_props(cbm_gbuf_t *gb, const char *qualified_name,
81+
const char *properties_json);
82+
7683
/* Find a node by qualified name. Returns NULL if not found. */
7784
const cbm_gbuf_node_t *cbm_gbuf_find_by_qn(const cbm_gbuf_t *gb, const char *qn);
7885

‎src/mcp/mcp.c‎

Lines changed: 91 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -316,7 +316,14 @@ static const tool_def_t TOOLS[] = {
316316
"Index a repository into the knowledge graph. "
317317
"Special mode 'cross-repo-intelligence': skip extraction, only match Routes/Channels "
318318
"across projects to create CROSS_HTTP_CALLS/CROSS_ASYNC_CALLS/CROSS_CHANNEL edges. "
319-
"Requires target_projects param. Ensure target projects have fresh indexes first.",
319+
"Requires target_projects param. Ensure target projects have fresh indexes first. "
320+
"COVERAGE: the response reports files that were NOT fully indexed — 'skipped' (not "
321+
"indexed at all: oversized/read/parse failures) and 'parse_partial' (indexed, but "
322+
"constructs inside the listed line ranges could not be parsed and are absent from the "
323+
"graph). parse_partial files also carry {parse_incomplete:true, error_ranges} on their "
324+
"File node — query later via query_graph: MATCH (f:File) WHERE f.parse_incomplete = true "
325+
"RETURN f. Both signals are best-effort: absence of a flag is NOT a completeness "
326+
"guarantee; prefer grep inside flagged ranges.",
320327
"{\"type\":\"object\",\"properties\":{\"repo_path\":{\"type\":\"string\",\"description\":"
321328
"\"Path to the repository\"},"
322329
"\"mode\":{\"type\":\"string\","
@@ -3299,36 +3306,100 @@ static void add_excluded_summary(yyjson_mut_doc *doc, yyjson_mut_val *root, char
32993306
* the JSON carries "count" + "truncated" so nothing is silently hidden. */
33003307
enum { INDEX_SKIPPED_FILE_CAP = 50 };
33013308

3309+
/* True when a recorded per-file entry is the parse-partial coverage signal
3310+
* (#963) rather than a genuine skip. Kept out of skipped[]/skipped_count so
3311+
* the "skipped" contract (file NOT indexed) stays exact. */
3312+
static bool is_parse_partial(const cbm_file_error_t *e) {
3313+
return e->phase && strcmp(e->phase, "parse_partial") == 0;
3314+
}
3315+
33023316
/* Attach a summary of per-file skips (Stage 2 / Track B). Always emits a
33033317
* top-level "skipped_count" (0 on clean runs) so consumers can rely on it.
33043318
* When there are skips, also emits:
33053319
* "skipped": {"files":[{path,reason,phase}..(<=50)], "count":N, "truncated":bool}
33063320
* and, if a per-run logfile was written, "logfile": "<path>".
33073321
* The run status stays "indexed" — a skipped file is the expected handled
3308-
* outcome, not a failure. errs[] is borrowed (copied into doc). */
3322+
* outcome, not a failure. errs[] is borrowed (copied into doc) and may contain
3323+
* parse_partial entries, which are filtered out here (reported separately by
3324+
* add_parse_partial_summary). */
33093325
static void add_skipped_summary(yyjson_mut_doc *doc, yyjson_mut_val *root,
33103326
const cbm_file_error_t *errs, int count, const char *logfile) {
3311-
yyjson_mut_obj_add_int(doc, root, "skipped_count", count < 0 ? 0 : count);
3312-
if (!errs || count <= 0) {
3327+
int skips = 0;
3328+
for (int i = 0; i < count; i++) {
3329+
if (!is_parse_partial(&errs[i])) {
3330+
skips++;
3331+
}
3332+
}
3333+
yyjson_mut_obj_add_int(doc, root, "skipped_count", skips);
3334+
if (logfile && logfile[0]) {
3335+
yyjson_mut_obj_add_strcpy(doc, root, "logfile", logfile);
3336+
}
3337+
if (!errs || skips <= 0) {
33133338
return;
33143339
}
33153340
yyjson_mut_val *skipped = yyjson_mut_obj(doc);
33163341
yyjson_mut_val *files = yyjson_mut_arr(doc);
3317-
int shown = count < INDEX_SKIPPED_FILE_CAP ? count : INDEX_SKIPPED_FILE_CAP;
3318-
for (int i = 0; i < shown; i++) {
3342+
int shown = 0;
3343+
for (int i = 0; i < count && shown < INDEX_SKIPPED_FILE_CAP; i++) {
3344+
if (is_parse_partial(&errs[i])) {
3345+
continue;
3346+
}
33193347
yyjson_mut_val *fe = yyjson_mut_obj(doc);
33203348
yyjson_mut_obj_add_strcpy(doc, fe, "path", errs[i].path ? errs[i].path : "");
33213349
yyjson_mut_obj_add_strcpy(doc, fe, "reason", errs[i].reason ? errs[i].reason : "");
33223350
yyjson_mut_obj_add_strcpy(doc, fe, "phase", errs[i].phase ? errs[i].phase : "");
33233351
yyjson_mut_arr_add_val(files, fe);
3352+
shown++;
33243353
}
33253354
yyjson_mut_obj_add_val(doc, skipped, "files", files);
3326-
yyjson_mut_obj_add_int(doc, skipped, "count", count);
3327-
yyjson_mut_obj_add_bool(doc, skipped, "truncated", count > INDEX_SKIPPED_FILE_CAP);
3355+
yyjson_mut_obj_add_int(doc, skipped, "count", skips);
3356+
yyjson_mut_obj_add_bool(doc, skipped, "truncated", skips > INDEX_SKIPPED_FILE_CAP);
33283357
yyjson_mut_obj_add_val(doc, root, "skipped", skipped);
3329-
if (logfile && logfile[0]) {
3330-
yyjson_mut_obj_add_strcpy(doc, root, "logfile", logfile);
3358+
}
3359+
3360+
/* Attach the best-effort parse-coverage summary (#963). Always emits a
3361+
* top-level "parse_partial_count" (0 on clean runs). When files were flagged:
3362+
* "parse_partial": {"files":[{path,error_ranges}..(<=50)], "count":N,
3363+
* "truncated":bool, "note":"..."}
3364+
* These files WERE indexed — constructs inside the listed 1-based line ranges
3365+
* are missing from the graph because tree-sitter could not parse them. The
3366+
* note spells out the best-effort framing: absence from this list is NOT a
3367+
* completeness guarantee. */
3368+
static void add_parse_partial_summary(yyjson_mut_doc *doc, yyjson_mut_val *root,
3369+
const cbm_file_error_t *errs, int count) {
3370+
int partials = 0;
3371+
for (int i = 0; i < count; i++) {
3372+
if (is_parse_partial(&errs[i])) {
3373+
partials++;
3374+
}
3375+
}
3376+
yyjson_mut_obj_add_int(doc, root, "parse_partial_count", partials);
3377+
if (!errs || partials <= 0) {
3378+
return;
33313379
}
3380+
yyjson_mut_val *pp = yyjson_mut_obj(doc);
3381+
yyjson_mut_val *files = yyjson_mut_arr(doc);
3382+
int shown = 0;
3383+
for (int i = 0; i < count && shown < INDEX_SKIPPED_FILE_CAP; i++) {
3384+
if (!is_parse_partial(&errs[i])) {
3385+
continue;
3386+
}
3387+
yyjson_mut_val *fe = yyjson_mut_obj(doc);
3388+
yyjson_mut_obj_add_strcpy(doc, fe, "path", errs[i].path ? errs[i].path : "");
3389+
yyjson_mut_obj_add_strcpy(doc, fe, "error_ranges", errs[i].reason ? errs[i].reason : "");
3390+
yyjson_mut_arr_add_val(files, fe);
3391+
shown++;
3392+
}
3393+
yyjson_mut_obj_add_val(doc, pp, "files", files);
3394+
yyjson_mut_obj_add_int(doc, pp, "count", partials);
3395+
yyjson_mut_obj_add_bool(doc, pp, "truncated", partials > INDEX_SKIPPED_FILE_CAP);
3396+
yyjson_mut_obj_add_str(doc, pp, "note",
3397+
"Best-effort signal, not a completeness guarantee: these files WERE "
3398+
"indexed, but constructs inside the listed line ranges (1-based) could "
3399+
"not be parsed and are missing from the graph. Prefer text search "
3400+
"(grep) for those regions. Files absent from this list are NOT "
3401+
"guaranteed to be fully indexed.");
3402+
yyjson_mut_obj_add_val(doc, root, "parse_partial", pp);
33323403
}
33333404

33343405
/* Write the FULL (uncapped) skip list to a per-run logfile — ONLY when >=1 file
@@ -3360,8 +3431,15 @@ static bool write_skip_logfile(const char *project, const cbm_file_error_t *errs
33603431
cbm_log_warn("index.logfile_open_fail", "path", path);
33613432
return false;
33623433
}
3363-
(void)fprintf(f, "# codebase-memory-mcp index skip report\n");
3364-
(void)fprintf(f, "# project=%s skipped=%d\n", project ? project : "", count);
3434+
int partials = 0;
3435+
for (int i = 0; i < count; i++) {
3436+
if (is_parse_partial(&errs[i])) {
3437+
partials++;
3438+
}
3439+
}
3440+
(void)fprintf(f, "# codebase-memory-mcp index coverage report\n");
3441+
(void)fprintf(f, "# project=%s skipped=%d parse_partial=%d\n", project ? project : "",
3442+
count - partials, partials);
33653443
(void)fprintf(f, "# columns: phase\treason\tpath\n");
33663444
for (int i = 0; i < count; i++) {
33673445
(void)fprintf(f, "%s\t%s\t%s\n", errs[i].phase ? errs[i].phase : "",
@@ -3384,6 +3462,7 @@ static bool build_index_success_response(cbm_mcp_server_t *srv, yyjson_mut_doc *
33843462
const char *logfile) {
33853463
add_excluded_summary(doc, root, excluded_dirs, excluded_count);
33863464
add_skipped_summary(doc, root, file_errors, file_error_count, logfile);
3465+
add_parse_partial_summary(doc, root, file_errors, file_error_count);
33873466

33883467
int exp_nodes = -1;
33893468
int exp_edges = -1;

‎src/pipeline/pass_definitions.c‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -581,6 +581,12 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
581581
result->error_msg ? result->error_msg : "extract failed",
582582
"extract");
583583
errors++;
584+
} else if (result->parse_incomplete) {
585+
/* Best-effort parse-coverage signal (#963): indexed, but with
586+
* ERROR/MISSING regions — see pass_parallel.c (keep in sync). */
587+
cbm_pipeline_add_file_error(ctx->pipeline, rel,
588+
result->error_ranges ? result->error_ranges : "unknown",
589+
"parse_partial");
584590
}
585591

586592
/* Create nodes for each definition */

‎src/pipeline/pass_parallel.c‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -902,6 +902,14 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
902902
pp_err_add(errs, fi->rel_path, result->error_msg ? result->error_msg : "extract failed",
903903
"extract");
904904
ws->errors++;
905+
} else if (result->parse_incomplete) {
906+
/* Best-effort parse-coverage signal (#963): the file WAS indexed,
907+
* but its tree contains ERROR/MISSING regions whose constructs are
908+
* silently absent from the graph. Not a skip — recorded under the
909+
* distinct "parse_partial" phase (reason = the line-range list) so
910+
* the MCP layer reports it separately from skipped[]. */
911+
pp_err_add(errs, fi->rel_path, result->error_ranges ? result->error_ranges : "unknown",
912+
"parse_partial");
905913
}
906914

907915
/* Create definition nodes in local gbuf */

‎src/pipeline/pipeline.c‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -315,6 +315,40 @@ void cbm_pipeline_get_file_errors(const cbm_pipeline_t *p, cbm_file_error_t **ou
315315
}
316316
}
317317

318+
void cbm_pipeline_stamp_parse_partial(cbm_pipeline_t *p, cbm_gbuf_t *gbuf) {
319+
if (!p || !gbuf) {
320+
return;
321+
}
322+
int stamped = 0;
323+
for (int i = 0; i < p->file_errors_count; i++) {
324+
const cbm_file_error_t *e = &p->file_errors[i];
325+
if (!e->path || !e->phase || strcmp(e->phase, "parse_partial") != 0) {
326+
continue;
327+
}
328+
char *file_qn = cbm_pipeline_fqn_compute(p->project_name, e->path, "__file__");
329+
if (!file_qn) {
330+
continue;
331+
}
332+
const char *slash = strrchr(e->path, '/');
333+
const char *basename = slash ? slash + SKIP_ONE : e->path;
334+
const char *ext = strrchr(basename, '.');
335+
/* Full replacement props (set_node_props does not merge): keep the
336+
* "extension" key pass_structure wrote, add the coverage marker. The
337+
* ranges string is digits/commas/dashes — JSON-safe unescaped. */
338+
char props[CBM_SZ_2K];
339+
snprintf(props, sizeof(props),
340+
"{\"extension\":\"%s\",\"parse_incomplete\":true,\"error_ranges\":\"%s\"}",
341+
ext ? ext : "", e->reason ? e->reason : "");
342+
if (cbm_gbuf_set_node_props(gbuf, file_qn, props)) {
343+
stamped++;
344+
}
345+
free(file_qn);
346+
}
347+
if (stamped > 0) {
348+
cbm_log_info("index.parse_partial", "files", itoa_buf(stamped));
349+
}
350+
}
351+
318352
void cbm_pipeline_get_committed_counts(const cbm_pipeline_t *p, int *nodes, int *edges) {
319353
if (nodes) {
320354
*nodes = p ? p->committed_nodes : -1;
@@ -1262,6 +1296,11 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
12621296
if (check_cancel(p)) {
12631297
return CBM_NOT_FOUND;
12641298
}
1299+
if (rc == 0) {
1300+
/* Extraction merged its per-file errors — stamp parse-partial File
1301+
* nodes (#963) before downstream passes and the dump. */
1302+
cbm_pipeline_stamp_parse_partial(p, p->gbuf);
1303+
}
12651304
return rc;
12661305
}
12671306

0 commit comments

Comments
 (0)