Skip to content

Commit e89aa77

Browse files
committed
feat(index): report per-file indexing failures via skipped[] + logfile (Stage 2/B2-B4)
index_repository silently dropped files that failed to index: CBMFileResult.has_error was set but never read, oversized files (>100 MB) were dropped with no signal, and read/extract failures only bumped a logged-only counter — a file that couldn't be indexed just vanished from the graph. Collect and surface them: - New retained {path, reason, phase} error list on struct cbm_pipeline (mirrors the excluded_dirs pattern) + accessor + a back-pointer on cbm_pipeline_ctx_t so both extraction paths can append. Wired in BOTH the sequential (pass_definitions) and parallel (per-worker, merged lock-free) paths — small repos take the sequential path, so wiring only one would leave the guard vacuous. - Feeds: read-fail, extract-fail, the newly-CONSUMED has_error (parse timeout / parse failed, with error_msg), and oversized. The cross_lsp phase is reserved for the crash supervisor (Track C) and not fed here (the cross-LSP passes are best-effort with no failure return; feeding the no-source case would be false positives). - MCP/CLI response gains top-level "skipped_count" (0 on clean) and, when >0, a capped "skipped":{files[<=50],count,truncated} + "logfile". Status stays "indexed" — a reported skip is the expected, handled outcome, not a degradation. - Per-run logfile (full uncapped list) written ONLY when >=1 file is skipped: $CBM_INDEX_LOG override else <cache_dir>/logs/<project>-<epoch>.log. - Generous env-configurable caps (src/foundation/limits.c): CBM_MAX_FILE_BYTES, default raised 100 MB -> 512 MiB; over-cap files are REPORTED (phase oversized) + WARNed, never silently dropped. Reproduce-first: tests/test_index_resilience.c (gating) — an oversized file (cap lowered via env) must appear in skipped[] with the 2 good files still indexed and a logfile written; a clean run has skipped_count 0 and no logfile. Genuine guard: no-op'ing the recorder makes the oversized file silently vanish (RED). Full suite 5750/0, no ASan/UBSan. Part of the resilient-indexing effort (Track B surfacing layer). Refs DeusData#668. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent dcf98dc commit e89aa77

16 files changed

Lines changed: 747 additions & 19 deletions

‎Makefile.cbm‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,8 @@ FOUNDATION_SRCS = \
112112
src/foundation/mem.c \
113113
src/foundation/diagnostics.c \
114114
src/foundation/profile.c \
115-
src/foundation/dump_verify.c
115+
src/foundation/dump_verify.c \
116+
src/foundation/limits.c
116117

117118
# Existing extraction C code (compiled from current location)
118119
EXTRACTION_SRCS = \
@@ -345,7 +346,7 @@ TEST_DISCOVER_SRCS = \
345346

346347
TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c
347348

348-
TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c
349+
TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c
349350

350351
TEST_WATCHER_SRCS = tests/test_watcher.c
351352

@@ -567,7 +568,7 @@ $(UNIXCODER_OBJ): $(UNIXCODER_BLOB_SRC) vendored/nomic/code_vectors.bin | $(BUIL
567568
OBJS_VENDORED_TEST = $(MIMALLOC_OBJ_TEST) $(SQLITE3_OBJ_TEST) $(TRE_OBJ_TEST) $(GRAMMAR_OBJS_TEST) $(TS_RUNTIME_OBJ_TEST) $(LSP_OBJ_TEST) $(PP_OBJ_TEST) $(LZ4_OBJ_TEST) $(ZSTD_OBJ_TEST) $(UNIXCODER_OBJ)
568569

569570
$(BUILD_DIR)/test-runner: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) | $(BUILD_DIR)
570-
$(CC) $(CFLAGS_TEST) -o $@ \
571+
$(CC) $(CFLAGS_TEST) -Itests -Itests/repro -o $@ \
571572
$(ALL_TEST_SRCS) $(PROD_SRCS) \
572573
$(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \
573574
$(OBJS_VENDORED_TEST) \

‎src/foundation/limits.c‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* limits.c — Env-configurable safety limits (Stage 2 / Track B4).
3+
*/
4+
#include "foundation/limits.h"
5+
6+
#include <errno.h>
7+
#include <stdlib.h>
8+
9+
long cbm_max_file_bytes(void) {
10+
/* 512 MiB — generous: real source files never approach it, but a
11+
* pathological / vendored blob degrades to a reported "oversized" skip
12+
* instead of a silent drop or an unbounded read. */
13+
const long default_cap = 512L * 1024 * 1024;
14+
15+
const char *raw = getenv("CBM_MAX_FILE_BYTES");
16+
if (raw && raw[0]) {
17+
errno = 0;
18+
char *end = NULL;
19+
long v = strtol(raw, &end, 10);
20+
if (errno == 0 && end != raw && *end == '\0' && v > 0) {
21+
return v;
22+
}
23+
/* Unparseable / non-positive → fall through to the safe default. */
24+
}
25+
return default_cap;
26+
}

‎src/foundation/limits.h‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* limits.h — Generous, env-configurable safety limits (Stage 2 / Track B4).
3+
*
4+
* Each knob has a generous default. Hitting a limit degrades to a *reported*
5+
* skip (surfaced via MCP/CLI/logfile), never a silent drop and never an
6+
* unbounded read (unbounded just trades a crash for an OOM/hang). Every limit
7+
* is env-overridable so an operator can tune it per-repo without a rebuild.
8+
*/
9+
#ifndef CBM_LIMITS_H
10+
#define CBM_LIMITS_H
11+
12+
/* Result of an attempted per-file read, so callers can attribute a skip to the
13+
* right phase/reason instead of collapsing every failure into "read failed". */
14+
typedef enum {
15+
CBM_READ_OK = 0, /* file read successfully */
16+
CBM_READ_OPEN_FAIL, /* could not open (missing / permission) */
17+
CBM_READ_EMPTY, /* zero/negative size — benign, nothing to index */
18+
CBM_READ_OVERSIZED, /* size exceeds cbm_max_file_bytes() */
19+
CBM_READ_OOM, /* buffer allocation failed */
20+
} cbm_read_status_t;
21+
22+
/* Maximum size (bytes) of a single source file the indexer will read into
23+
* memory. Files larger than this are skipped-and-reported (phase "oversized"),
24+
* never silently dropped. Override with CBM_MAX_FILE_BYTES (a positive integer
25+
* count of bytes). Default 512 MiB (raised from the historical 100 MB cap).
26+
*
27+
* The env var is read on each call — this is intentional: read_file() calls it
28+
* once per file (negligible), and reading fresh means a test / operator can
29+
* change the cap via setenv without a process restart or a stale memoized copy
30+
* leaking across runs. */
31+
long cbm_max_file_bytes(void);
32+
33+
#endif /* CBM_LIMITS_H */

‎src/mcp/mcp.c‎

Lines changed: 99 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3134,13 +3134,96 @@ static void add_excluded_summary(yyjson_mut_doc *doc, yyjson_mut_val *root, char
31343134
yyjson_mut_obj_add_val(doc, root, "excluded", excluded);
31353135
}
31363136

3137+
/* Cap on per-file skips embedded in the JSON response — keep it compact on
3138+
* large repos. The FULL, uncapped list always goes to the per-run logfile;
3139+
* the JSON carries "count" + "truncated" so nothing is silently hidden. */
3140+
enum { INDEX_SKIPPED_FILE_CAP = 50 };
3141+
3142+
/* Attach a summary of per-file skips (Stage 2 / Track B). Always emits a
3143+
* top-level "skipped_count" (0 on clean runs) so consumers can rely on it.
3144+
* When there are skips, also emits:
3145+
* "skipped": {"files":[{path,reason,phase}..(<=50)], "count":N, "truncated":bool}
3146+
* and, if a per-run logfile was written, "logfile": "<path>".
3147+
* The run status stays "indexed" — a skipped file is the expected handled
3148+
* outcome, not a failure. errs[] is borrowed (copied into doc). */
3149+
static void add_skipped_summary(yyjson_mut_doc *doc, yyjson_mut_val *root,
3150+
const cbm_file_error_t *errs, int count, const char *logfile) {
3151+
yyjson_mut_obj_add_int(doc, root, "skipped_count", count < 0 ? 0 : count);
3152+
if (!errs || count <= 0) {
3153+
return;
3154+
}
3155+
yyjson_mut_val *skipped = yyjson_mut_obj(doc);
3156+
yyjson_mut_val *files = yyjson_mut_arr(doc);
3157+
int shown = count < INDEX_SKIPPED_FILE_CAP ? count : INDEX_SKIPPED_FILE_CAP;
3158+
for (int i = 0; i < shown; i++) {
3159+
yyjson_mut_val *fe = yyjson_mut_obj(doc);
3160+
yyjson_mut_obj_add_strcpy(doc, fe, "path", errs[i].path ? errs[i].path : "");
3161+
yyjson_mut_obj_add_strcpy(doc, fe, "reason", errs[i].reason ? errs[i].reason : "");
3162+
yyjson_mut_obj_add_strcpy(doc, fe, "phase", errs[i].phase ? errs[i].phase : "");
3163+
yyjson_mut_arr_add_val(files, fe);
3164+
}
3165+
yyjson_mut_obj_add_val(doc, skipped, "files", files);
3166+
yyjson_mut_obj_add_int(doc, skipped, "count", count);
3167+
yyjson_mut_obj_add_bool(doc, skipped, "truncated", count > INDEX_SKIPPED_FILE_CAP);
3168+
yyjson_mut_obj_add_val(doc, root, "skipped", skipped);
3169+
if (logfile && logfile[0]) {
3170+
yyjson_mut_obj_add_strcpy(doc, root, "logfile", logfile);
3171+
}
3172+
}
3173+
3174+
/* Write the FULL (uncapped) skip list to a per-run logfile — ONLY when >=1 file
3175+
* was skipped (no logfile on a clean run). Location:
3176+
* $CBM_INDEX_LOG (override) else <cache_dir>/logs/<project>-<epoch>.log
3177+
* Returns true and fills out_path on success. */
3178+
static bool write_skip_logfile(const char *project, const cbm_file_error_t *errs, int count,
3179+
char *out_path, size_t out_sz) {
3180+
if (!errs || count <= 0) {
3181+
return false;
3182+
}
3183+
char path[CBM_SZ_1K];
3184+
const char *override = getenv("CBM_INDEX_LOG");
3185+
if (override && override[0]) {
3186+
snprintf(path, sizeof(path), "%s", override);
3187+
} else {
3188+
const char *cdir = cbm_resolve_cache_dir();
3189+
if (!cdir) {
3190+
return false;
3191+
}
3192+
char logdir[CBM_SZ_1K];
3193+
snprintf(logdir, sizeof(logdir), "%s/logs", cdir);
3194+
cbm_mkdir_p(logdir, 0755);
3195+
snprintf(path, sizeof(path), "%s/%s-%lld.log", logdir, project ? project : "index",
3196+
(long long)time(NULL));
3197+
}
3198+
FILE *f = cbm_fopen(path, "wb");
3199+
if (!f) {
3200+
cbm_log_warn("index.logfile_open_fail", "path", path);
3201+
return false;
3202+
}
3203+
(void)fprintf(f, "# codebase-memory-mcp index skip report\n");
3204+
(void)fprintf(f, "# project=%s skipped=%d\n", project ? project : "", count);
3205+
(void)fprintf(f, "# columns: phase\treason\tpath\n");
3206+
for (int i = 0; i < count; i++) {
3207+
(void)fprintf(f, "%s\t%s\t%s\n", errs[i].phase ? errs[i].phase : "",
3208+
errs[i].reason ? errs[i].reason : "", errs[i].path ? errs[i].path : "");
3209+
}
3210+
(void)fclose(f);
3211+
if (out_path && out_sz) {
3212+
snprintf(out_path, out_sz, "%s", path);
3213+
}
3214+
return true;
3215+
}
3216+
31373217
/* Build the success portion of the index_repository response.
31383218
* Returns true when status should be "degraded" (#334 plausibility gate). */
31393219
static bool build_index_success_response(cbm_mcp_server_t *srv, yyjson_mut_doc *doc,
31403220
yyjson_mut_val *root, const char *project_name,
31413221
const char *repo_path, bool persistence, cbm_pipeline_t *p,
3142-
char **excluded_dirs, int excluded_count) {
3222+
char **excluded_dirs, int excluded_count,
3223+
const cbm_file_error_t *file_errors, int file_error_count,
3224+
const char *logfile) {
31433225
add_excluded_summary(doc, root, excluded_dirs, excluded_count);
3226+
add_skipped_summary(doc, root, file_errors, file_error_count, logfile);
31443227

31453228
int exp_nodes = -1;
31463229
int exp_edges = -1;
@@ -3302,6 +3385,12 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
33023385
int excluded_count = 0;
33033386
cbm_pipeline_get_excluded(p, &excluded_dirs, &excluded_count);
33043387

3388+
/* Capture the per-file skip list (Stage 2 / Track B) while the pipeline
3389+
* still owns the strings; the response builder copies them into the doc. */
3390+
cbm_file_error_t *file_errors = NULL;
3391+
int file_error_count = 0;
3392+
cbm_pipeline_get_file_errors(p, &file_errors, &file_error_count);
3393+
33053394
cbm_mem_collect(); /* return mimalloc pages to OS after large indexing */
33063395

33073396
/* Invalidate cached store so next query reopens the fresh database */
@@ -3319,8 +3408,15 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
33193408
yyjson_mut_obj_add_str(doc, root, "project", project_name);
33203409

33213410
if (rc == 0) {
3322-
bool degraded = build_index_success_response(srv, doc, root, project_name, repo_path,
3323-
persistence, p, excluded_dirs, excluded_count);
3411+
/* Write the per-run logfile ONLY when there were skips (no logfile on a
3412+
* clean run). The FULL list goes to the file; the JSON caps at 50. */
3413+
char logfile_path[CBM_SZ_1K];
3414+
logfile_path[0] = '\0';
3415+
bool has_logfile = write_skip_logfile(project_name, file_errors, file_error_count,
3416+
logfile_path, sizeof(logfile_path));
3417+
bool degraded = build_index_success_response(
3418+
srv, doc, root, project_name, repo_path, persistence, p, excluded_dirs, excluded_count,
3419+
file_errors, file_error_count, has_logfile ? logfile_path : NULL);
33243420
yyjson_mut_obj_add_str(doc, root, "status", degraded ? "degraded" : "indexed");
33253421
} else {
33263422
yyjson_mut_obj_add_str(doc, root, "status", "error");

‎src/pipeline/pass_calls.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ enum { PC_RING = 4, PC_RING_MASK = 3, PC_SIG_SCAN = 15, PC_REGEX_GRP = 2 };
2323
#include "foundation/log.h"
2424
#include "foundation/compat.h"
2525
#include "foundation/compat_fs.h"
26+
#include "foundation/limits.h"
2627
#include "foundation/str_util.h"
2728
#include "cbm.h"
2829
#include "service_patterns.h"
@@ -53,7 +54,7 @@ static char *read_file(const char *path, int *out_len) {
5354
long size = ftell(f);
5455
(void)fseek(f, 0, SEEK_SET);
5556

56-
if (size <= 0 || size > (long)CBM_PERCENT * CBM_SZ_1K * CBM_SZ_1K) {
57+
if (size <= 0 || size > cbm_max_file_bytes()) { /* generous, env-configurable cap (B4) */
5758
(void)fclose(f);
5859
return NULL;
5960
}

‎src/pipeline/pass_definitions.c‎

Lines changed: 63 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ enum { PD_JSON_FIELD_OVERHEAD = 6 };
2323
#include "foundation/log.h"
2424
#include "foundation/compat.h"
2525
#include "foundation/compat_fs.h"
26+
#include "foundation/limits.h"
2627
#include "cbm.h"
2728
#include "simhash/minhash.h"
2829
#include "semantic/ast_profile.h"
@@ -32,20 +33,45 @@ enum { PD_JSON_FIELD_OVERHEAD = 6 };
3233
#include <string.h>
3334

3435
/* Read entire file into heap-allocated buffer. Returns NULL on error.
35-
* Caller must free(). Sets *out_len to byte count. */
36-
static char *read_file(const char *path, int *out_len) {
36+
* Caller must free(). Sets *out_len to byte count. *out_size receives the
37+
* on-disk size and *out_status the failure reason, so the caller can attribute
38+
* a skip to the right phase/reason (read vs oversized) instead of a silent
39+
* drop. Both out params may be NULL. */
40+
static char *read_file(const char *path, int *out_len, long *out_size,
41+
cbm_read_status_t *out_status) {
42+
if (out_size) {
43+
*out_size = 0;
44+
}
45+
if (out_status) {
46+
*out_status = CBM_READ_OK;
47+
}
3748
FILE *f = cbm_fopen(path, "rb");
3849
if (!f) {
50+
if (out_status) {
51+
*out_status = CBM_READ_OPEN_FAIL;
52+
}
3953
return NULL;
4054
}
4155

4256
(void)fseek(f, 0, SEEK_END);
4357
long size = ftell(f);
4458
(void)fseek(f, 0, SEEK_SET);
59+
if (out_size) {
60+
*out_size = size;
61+
}
4562

46-
if (size <= 0 ||
47-
size > (long)CBM_PERCENT * CBM_SZ_1K * CBM_SZ_1K) { /* CBM_PERCENT MB sanity limit */
63+
if (size <= 0) {
64+
(void)fclose(f);
65+
if (out_status) {
66+
*out_status = CBM_READ_EMPTY;
67+
}
68+
return NULL;
69+
}
70+
if (size > cbm_max_file_bytes()) { /* generous, env-configurable cap (B4) */
4871
(void)fclose(f);
72+
if (out_status) {
73+
*out_status = CBM_READ_OVERSIZED;
74+
}
4975
return NULL;
5076
}
5177

@@ -57,6 +83,9 @@ static char *read_file(const char *path, int *out_len) {
5783
char *buf = malloc((size_t)size + CBM_TS_LOOKAHEAD_PAD);
5884
if (!buf) {
5985
(void)fclose(f);
86+
if (out_status) {
87+
*out_status = CBM_READ_OOM;
88+
}
6089
return NULL;
6190
}
6291

@@ -487,9 +516,27 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
487516

488517
/* Read source file */
489518
int source_len = 0;
490-
char *source = read_file(path, &source_len);
519+
long file_size = 0;
520+
cbm_read_status_t rst = CBM_READ_OK;
521+
char *source = read_file(path, &source_len, &file_size, &rst);
491522
if (!source) {
492523
errors++;
524+
if (rst == CBM_READ_OVERSIZED) {
525+
/* Never a silent drop: record the oversized skip + WARN so the
526+
* file surfaces in the response/logfile with its sizes. */
527+
long cap = cbm_max_file_bytes();
528+
char reason[96];
529+
snprintf(reason, sizeof(reason), "oversized (%lld MB > %lld MB)",
530+
(long long)(file_size / (CBM_SZ_1K * CBM_SZ_1K)),
531+
(long long)(cap / (CBM_SZ_1K * CBM_SZ_1K)));
532+
cbm_pipeline_add_file_error(ctx->pipeline, rel, reason, "oversized");
533+
cbm_log_warn("index.file_oversized", "path", rel, "size_mb",
534+
itoa_log((int)(file_size / (CBM_SZ_1K * CBM_SZ_1K))), "cap_mb",
535+
itoa_log((int)(cap / (CBM_SZ_1K * CBM_SZ_1K))));
536+
} else if (rst == CBM_READ_OPEN_FAIL || rst == CBM_READ_OOM) {
537+
cbm_pipeline_add_file_error(ctx->pipeline, rel, "read failed", "read");
538+
}
539+
/* CBM_READ_EMPTY: benign 0-byte file — nothing to index, not reported. */
493540
continue;
494541
}
495542

@@ -502,8 +549,19 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
502549

503550
if (!result) {
504551
errors++;
552+
cbm_pipeline_add_file_error(ctx->pipeline, rel, "extract failed", "extract");
505553
continue;
506554
}
555+
/* Consume the previously-ignored has_error flag: a parse timeout /
556+
* parse failure / unsupported-grammar result carries no defs but must
557+
* still be reported (phase "extract", reason = the extractor's message).
558+
* The empty result flows through unchanged (the defs loop is a no-op). */
559+
if (result->has_error) {
560+
cbm_pipeline_add_file_error(ctx->pipeline, rel,
561+
result->error_msg ? result->error_msg : "extract failed",
562+
"extract");
563+
errors++;
564+
}
507565

508566
/* Create nodes for each definition */
509567
for (int d = 0; d < result->defs.count; d++) {

‎src/pipeline/pass_k8s.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "foundation/log.h"
2121
#include "foundation/compat.h"
2222
#include "foundation/compat_fs.h"
23+
#include "foundation/limits.h"
2324
#include "cbm.h"
2425

2526
#include <stdlib.h>
@@ -40,7 +41,7 @@ static char *k8s_read_file(const char *path, int *out_len) {
4041
long size = ftell(f);
4142
(void)fseek(f, 0, SEEK_SET);
4243

43-
if (size <= 0 || size > (long)CBM_PERCENT * CBM_SZ_1K * CBM_SZ_1K) {
44+
if (size <= 0 || size > cbm_max_file_bytes()) { /* generous, env-configurable cap (B4) */
4445
(void)fclose(f);
4546
return NULL;
4647
}

0 commit comments

Comments
 (0)