Skip to content

Commit 8758668

Browse files
committed
fix: resolve PR #18 module, memlab, snippet and recovery findings
Preserve path-derived JS/TS Module definitions through junk-name filtering so default-export axios metadata stays on the Module instead of a function. Remove the unused CompletedProcess.get access from the memlab driver. Measure snippet source prefixes with their final clipping/continuation shape, replace conflicting keys, update end_line, and restore the best fitting whole-line prefix after binary search. Move the recovery single-thread override into the shared worker-count policy so k8s/configlink helpers and automatic LSP-surface dispatch honor it even when CBM_WORKERS requests multiple workers. Remove the private duplicate. Add real-filesystem extraction, operation-entry snippet, full Kubernetes index, and isolated real-CLI/daemon memlab integration regressions. Validation: production build, -Werror test-runner build, CI lint, 748 passing extraction/integration/index_resilience/pipeline tests, 62 passing final integration/system_info/worker_pool checks, memlab E2E and Python compilation. Existing vendored ObjectScript UBSan warnings remain visible and unchanged. Also adds a `make -f Makefile.cbm test-memlab-drive` target for the new real-binary memlab check, alongside test-daemon-smoke. Addresses CodeRabbit review threads on PR #18: PRRT_kwDORjrg5c6ozbhf, PRRT_kwDORjrg5c6ozbh8, PRRT_kwDORjrg5c6ozbiA, PRRT_kwDORjrg5c6ozbiC.
1 parent 0311189 commit 8758668

10 files changed

Lines changed: 267 additions & 37 deletions

File tree

‎Makefile.cbm‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -867,7 +867,7 @@ PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o
867867

868868
# ── Targets ──────────────────────────────────────────────────────
869869

870-
.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security fuzz
870+
.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke test-memlab-drive cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security fuzz
871871

872872
$(BUILD_DIR):
873873
mkdir -p $(BUILD_DIR)
@@ -1141,6 +1141,11 @@ test-tsan: $(BUILD_DIR)/test-runner-tsan
11411141
test-daemon-smoke: $(BUILD_DIR)/codebase-memory-cli
11421142
cd $(CURDIR) && CBM_DAEMON_SMOKE_REQUIRE_RUN=1 python3 tests/test_daemon_smoke.py $(BUILD_DIR)/codebase-memory-cli
11431143

1144+
# Real-binary memlab driver check: indexes a temporary corpus through the CLI and
1145+
# an isolated daemon (private cache and runtime dirs), then serves two operations.
1146+
test-memlab-drive: $(BUILD_DIR)/codebase-memory-cli
1147+
cd $(CURDIR) && python3 tests/integration/test_memlab_drive.py $(BUILD_DIR)/codebase-memory-cli
1148+
11441149
# ── Production binary ────────────────────────────────────────────
11451150

11461151
# Grammar/TS/LSP objects for production (compiled with relaxed warnings, -O2)

‎internal/cbm/cbm.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,8 @@ static bool cbm_js_name_is_junk(const char *name) {
196196
void cbm_defs_push(CBMDefArray *arr, CBMArena *a, CBMDefinition def) {
197197
def.name = cbm_first_line(a, def.name);
198198
def.qualified_name = cbm_first_line(a, def.qualified_name);
199-
if (cbm_js_family_path(def.file_path) && cbm_js_name_is_junk(def.name)) {
199+
if (cbm_js_family_path(def.file_path) && !(def.label && strcmp(def.label, "Module") == 0) &&
200+
cbm_js_name_is_junk(def.name)) {
200201
return;
201202
}
202203
GROW_ARRAY(arr, a);

‎scripts/memlab-drive.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,6 @@ def main():
6969
if indexed.returncode != 0:
7070
print(f"index failed: {indexed.stderr.strip()}", file=sys.stderr)
7171
return 3
72-
project = (indexed.get("result", {}).get("structuredContent") or {}).get("project")
7372

7473
halfway = args.requests // 2
7574
argv = operation_args(args.tool)

‎src/foundation/platform.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ size_t cbm_system_available_ram(void);
127127
size_t cbm_fs_free_bytes(const char *path);
128128

129129
/* Recommended worker count for parallel indexing.
130+
* CBM_INDEX_SINGLE_THREAD=1 overrides CBM_WORKERS and detection for recovery.
130131
* initial=true: all cores (user is waiting for initial index)
131132
* initial=false: max(1, perf_cores-1) (leave headroom for user apps) */
132133
int cbm_default_worker_count(bool initial);

‎src/foundation/system_info.c‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,12 @@ cbm_system_info_t cbm_system_info(void) {
316316
}
317317

318318
int cbm_default_worker_count(bool initial) {
319+
/* Recovery must override even an explicit CBM_WORKERS setting in every
320+
* parallel helper, including auto-sized cbm_parallel_for dispatch. */
321+
const char *single = getenv("CBM_INDEX_SINGLE_THREAD");
322+
if (single && single[0] == '1') {
323+
return 1;
324+
}
319325
/* CBM_WORKERS env override (clamped to [1, CBM_WORKERS_MAX]).
320326
* Useful inside containers where sysconf(_SC_NPROCESSORS_ONLN)
321327
* reports host CPUs rather than the cgroup's effective CPU quota.

‎src/operations/snippet.c‎

Lines changed: 24 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,28 @@ static int snippet_source_line_count(const char *source) {
267267
return lines;
268268
}
269269

270+
/* Replace the whole response shape before measuring a source-prefix candidate. */
271+
static void snippet_set_source_prefix(yyjson_mut_doc *doc, yyjson_mut_val *root, const char *source,
272+
int lines, int start_line, int original_end) {
273+
size_t keep = 0;
274+
int line = 0;
275+
for (const char *q = source; lines > 0 && *q; q++) {
276+
keep++;
277+
if (*q == '\n' && ++line >= lines)
278+
break;
279+
}
280+
static const char *const keys[] = {"source", "source_truncated", "source_clipped",
281+
"next_start_line", "original_end_line", "end_line"};
282+
for (size_t i = 0; i < sizeof(keys) / sizeof(keys[0]); i++)
283+
(void)yyjson_mut_obj_remove_key(root, keys[i]);
284+
yyjson_mut_obj_add_val(doc, root, "source", yyjson_mut_strncpy(doc, source, keep));
285+
yyjson_mut_obj_add_int(doc, root, "end_line", lines > 0 ? start_line + lines - 1 : start_line);
286+
yyjson_mut_obj_add_bool(doc, root, "source_truncated", true);
287+
yyjson_mut_obj_add_bool(doc, root, "source_clipped", true);
288+
yyjson_mut_obj_add_int(doc, root, "next_start_line", start_line + lines);
289+
yyjson_mut_obj_add_int(doc, root, "original_end_line", original_end);
290+
}
291+
270292
/* The floor is metadata only: no partial line or identifier is ever emitted. */
271293
static char *snippet_budget_floor(int max_output_tokens) {
272294
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
@@ -521,21 +543,7 @@ static cbm_operation_result_t node_result(cbm_store_t *store, const char *projec
521543
int best_lines = -1;
522544
while (low <= high) {
523545
int middle = low + (high - low) / 2;
524-
size_t keep = 0;
525-
if (middle > 0) {
526-
int line = 0;
527-
for (const char *q = source; *q; q++) {
528-
keep++;
529-
if (*q == '\n' && ++line >= middle) {
530-
break;
531-
}
532-
}
533-
}
534-
yyjson_mut_val *value = yyjson_mut_strncpy(doc, source, keep);
535-
if (value) {
536-
(void)yyjson_mut_obj_remove_key(root, "source");
537-
yyjson_mut_obj_add_val(doc, root, "source", value);
538-
}
546+
snippet_set_source_prefix(doc, root, source, middle, start_line, original_end);
539547
if (snippet_probe_len(root) <= byte_budget) {
540548
best_lines = middle;
541549
low = middle + 1;
@@ -544,10 +552,7 @@ static cbm_operation_result_t node_result(cbm_store_t *store, const char *projec
544552
}
545553
}
546554
if (best_lines >= 0) {
547-
yyjson_mut_obj_add_bool(doc, root, "source_truncated", true);
548-
yyjson_mut_obj_add_bool(doc, root, "source_clipped", true);
549-
yyjson_mut_obj_add_int(doc, root, "next_start_line", start_line + best_lines);
550-
yyjson_mut_obj_add_int(doc, root, "original_end_line", original_end);
555+
snippet_set_source_prefix(doc, root, source, best_lines, start_line, original_end);
551556
}
552557
}
553558
if (snippet_probe_len(root) > byte_budget) {

‎src/pipeline/pipeline.c‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -615,19 +615,6 @@ void cbm_pipeline_set_committed_counts(cbm_pipeline_t *p, int nodes, int edges)
615615
}
616616
}
617617

618-
/* Effective worker count. The crash supervisor re-runs its worker single-
619-
* threaded (CBM_INDEX_SINGLE_THREAD=1) so a per-file marker can pin the EXACT
620-
* crasher; a parallel re-run would race the marker. Honour that override
621-
* everywhere the worker count drives the parallel/sequential decision, so the
622-
* whole extraction phase collapses to the deterministic sequential path. */
623-
static int effective_worker_count(bool initial) {
624-
const char *st = getenv("CBM_INDEX_SINGLE_THREAD");
625-
if (st && st[0] == '1') {
626-
return 1;
627-
}
628-
return cbm_default_worker_count(initial);
629-
}
630-
631618
/* Resolve the DB path for this pipeline. Caller must free(). */
632619
static char *resolve_db_path(const cbm_pipeline_t *p) {
633620
if (!p) {
@@ -2623,7 +2610,7 @@ static int run_githistory(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) {
26232610
gh_compute_arg_t gh_arg = {.repo_path = ctx->repo_path, .result = &gh_result};
26242611

26252612
if (p->mode != CBM_MODE_FAST) {
2626-
if (effective_worker_count(true) > SKIP_ONE) {
2613+
if (cbm_default_worker_count(true) > SKIP_ONE) {
26272614
if (cbm_thread_create(&gh_thread, 0, gh_compute_thread_fn, &gh_arg) == 0) {
26282615
gh_threaded = true;
26292616
}
@@ -2723,7 +2710,7 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
27232710
return CBM_NOT_FOUND;
27242711
}
27252712

2726-
int worker_count = effective_worker_count(true);
2713+
int worker_count = cbm_default_worker_count(true);
27272714
CBM_PROF_START(t_extract_total);
27282715
int rc = (worker_count > SKIP_ONE && file_count > MIN_FILES_FOR_PARALLEL)
27292716
? run_parallel_pipeline(p, ctx, files, file_count, worker_count, &t)
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/usr/bin/env python3
2+
"""Run memlab-drive's indexing path against the real CLI and daemon."""
3+
import os
4+
from pathlib import Path
5+
import subprocess
6+
import sys
7+
import tempfile
8+
9+
repo = Path(__file__).resolve().parents[2]
10+
binary = Path(sys.argv[1] if len(sys.argv) > 1 else repo / 'build/c/codebase-memory-cli').resolve()
11+
with tempfile.TemporaryDirectory(prefix='cbm_memlab_test_') as tmp:
12+
root = Path(tmp)
13+
corpus = root / 'corpus'
14+
corpus.mkdir()
15+
(corpus / 'widget.py').write_text('def Widget():\n return 1\n', encoding='utf-8')
16+
env = dict(os.environ, CBM_CACHE_DIR=str(root / 'cache'),
17+
CBM_RUNTIME_DIR=str(root / 'runtime'))
18+
(root / 'runtime').mkdir(mode=0o700)
19+
reply = subprocess.run(
20+
[sys.executable, str(repo / 'scripts/memlab-drive.py'), str(binary), str(corpus),
21+
'2', '--tool', 'projects'], env=env, capture_output=True, text=True, timeout=60,
22+
)
23+
assert reply.returncode == 0, reply.stderr
24+
assert reply.stdout.strip().endswith('served=2 failed=0'), reply.stdout

‎tests/test_integration.c‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
#include "test_helpers.h"
1313
#include "test_operation_host.h"
1414
#include "operations/output_budget.h"
15+
#include "operations/result_wire.h"
16+
#include "cbm.h"
1517
#include <yyjson/yyjson.h>
1618
#include <store/store.h>
1719
#include <pipeline/pipeline.h>
@@ -106,6 +108,14 @@ static int create_test_project(void) {
106108
fprintf(f, " budgetMarker%d();\n", i);
107109
}
108110
fprintf(f, "}\n");
111+
fprintf(f, "\nfunction budgetSnippet() {\n");
112+
for (int i = 0; i < 40; i++) {
113+
fprintf(f,
114+
" // line %02d: a long source line to exercise whole-line output budgeting "
115+
"abcdefghijklmnopqrstuvwxyz\n",
116+
i);
117+
}
118+
fprintf(f, "}\n");
109119
fclose(f);
110120

111121
return 0;
@@ -544,6 +554,137 @@ TEST(integ_mcp_search_code_match_limit_and_budget) {
544554
PASS();
545555
}
546556

557+
TEST(integ_js_path_module_default_client) {
558+
char tmp[] = "/tmp/cbm_js_module_XXXXXX";
559+
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
560+
const char *paths[] = {".eslintrc.js", ".storybook/main.ts", "(group)/page.tsx"};
561+
const CBMLanguage langs[] = {CBM_LANG_JAVASCRIPT, CBM_LANG_TYPESCRIPT, CBM_LANG_TSX};
562+
const char *text = "import axios from 'axios';\n"
563+
"function helper() { return 1; }\n"
564+
"export default axios.create({baseURL: 'https://example.test/api'});\n";
565+
bool ok = true;
566+
cbm_init();
567+
for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) {
568+
const char *path = TH_PATH(tmp, paths[i]);
569+
if (th_write_file(path, text) != 0) {
570+
ok = false;
571+
break;
572+
}
573+
char source[512];
574+
FILE *f = fopen(path, "rb");
575+
if (!f) {
576+
ok = false;
577+
break;
578+
}
579+
size_t len = fread(source, 1, sizeof(source) - 1, f);
580+
fclose(f);
581+
source[len] = '\0';
582+
CBMFileResult *r =
583+
cbm_extract_file(source, (int)len, langs[i], "path-module", paths[i], 0, NULL, NULL);
584+
if (!r || r->defs.count < 2) {
585+
ok = false;
586+
} else {
587+
CBMDefinition *module = &r->defs.items[0];
588+
ok = ok && strcmp(module->label, "Module") == 0 &&
589+
strcmp(module->name, paths[i]) == 0 && module->http_client &&
590+
strcmp(module->http_client, "axios") == 0 && module->http_base_url &&
591+
strcmp(module->http_base_url, "https://example.test/api") == 0;
592+
for (int j = 1; j < r->defs.count; j++) {
593+
ok = ok && !r->defs.items[j].http_client && !r->defs.items[j].http_base_url;
594+
}
595+
}
596+
cbm_free_result(r);
597+
}
598+
th_rmtree(tmp);
599+
ASSERT_TRUE(ok);
600+
PASS();
601+
}
602+
603+
TEST(integ_mcp_snippet_line_and_output_budget) {
604+
char args[1024];
605+
snprintf(args, sizeof(args),
606+
"{\"project\":\"%s\",\"qualified_name\":\"%s.app.budgetSnippet\","
607+
"\"max_lines\":24}",
608+
g_project, g_project);
609+
char *base = cbm_test_operation_execute(g_srv, "get_code_snippet", args);
610+
ASSERT_NOT_NULL(base);
611+
yyjson_doc *base_doc = yyjson_read(base, strlen(base), 0);
612+
ASSERT_NOT_NULL(base_doc);
613+
yyjson_val *base_root = yyjson_doc_get_root(base_doc);
614+
const char *source = yyjson_get_str(yyjson_obj_get(base_root, "source"));
615+
ASSERT_NOT_NULL(source);
616+
int start = (int)yyjson_get_int(yyjson_obj_get(base_root, "start_line"));
617+
int original_end = (int)yyjson_get_int(yyjson_obj_get(base_root, "original_end_line"));
618+
ASSERT_TRUE(original_end > start + 23);
619+
/* Sweep boundaries: both successful and failed final binary-search probes,
620+
* including metadata growth, must keep the largest complete prefix. */
621+
for (int tokens = 200; tokens <= 600; tokens += 7) {
622+
snprintf(args, sizeof(args),
623+
"{\"project\":\"%s\",\"qualified_name\":\"%s.app.budgetSnippet\","
624+
"\"max_lines\":24,\"max_output_tokens\":%d}",
625+
g_project, g_project, tokens);
626+
char *reply = cbm_test_operation_execute(g_srv, "get_code_snippet", args);
627+
ASSERT_NOT_NULL(reply);
628+
ASSERT_TRUE(strlen(reply) <= (size_t)tokens * CBM_OUTPUT_BYTES_PER_TOKEN);
629+
yyjson_doc *doc = yyjson_read(reply, strlen(reply), 0);
630+
ASSERT_NOT_NULL(doc);
631+
yyjson_val *root = yyjson_doc_get_root(doc);
632+
const char *prefix = yyjson_get_str(yyjson_obj_get(root, "source"));
633+
ASSERT_NOT_NULL(prefix);
634+
ASSERT_TRUE(strncmp(source, prefix, strlen(prefix)) == 0);
635+
int lines = 0;
636+
for (const char *q = prefix; *q; q++) {
637+
if (*q == '\n')
638+
lines++;
639+
}
640+
ASSERT_TRUE(lines > 0 && lines < 24);
641+
ASSERT_EQ((int)yyjson_get_int(yyjson_obj_get(root, "end_line")), start + lines - 1);
642+
ASSERT_EQ((int)yyjson_get_int(yyjson_obj_get(root, "next_start_line")), start + lines);
643+
ASSERT_EQ((int)yyjson_get_int(yyjson_obj_get(root, "original_end_line")), original_end);
644+
ASSERT_TRUE(yyjson_get_bool(yyjson_obj_get(root, "source_truncated")));
645+
/* Parseable JSON alone permits duplicate keys. Check every root key. */
646+
size_t idx, max;
647+
yyjson_val *key, *val;
648+
const char *keys[] = {"source", "source_truncated", "source_clipped",
649+
"next_start_line", "original_end_line", "end_line"};
650+
for (size_t k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) {
651+
int occurrences = 0;
652+
yyjson_obj_foreach(root, idx, max, key, val) {
653+
(void)val;
654+
if (strcmp(yyjson_get_str(key), keys[k]) == 0)
655+
occurrences++;
656+
}
657+
ASSERT_EQ(occurrences, 1);
658+
}
659+
/* One more whole line with its final metadata must exceed the ceiling. */
660+
size_t keep = strlen(prefix);
661+
const char *next_end = strchr(source + keep, '\n');
662+
ASSERT_NOT_NULL(next_end);
663+
yyjson_mut_doc *probe = yyjson_mut_doc_new(NULL);
664+
ASSERT_NOT_NULL(probe);
665+
yyjson_mut_val *candidate = yyjson_val_mut_copy(probe, root);
666+
ASSERT_NOT_NULL(candidate);
667+
yyjson_mut_doc_set_root(probe, candidate);
668+
(void)yyjson_mut_obj_remove_key(candidate, "source");
669+
yyjson_mut_obj_add_val(probe, candidate, "source",
670+
yyjson_mut_strncpy(probe, source, (size_t)(next_end - source + 1)));
671+
(void)yyjson_mut_obj_remove_key(candidate, "end_line");
672+
yyjson_mut_obj_add_int(probe, candidate, "end_line", start + lines);
673+
(void)yyjson_mut_obj_remove_key(candidate, "next_start_line");
674+
yyjson_mut_obj_add_int(probe, candidate, "next_start_line", start + lines + 1);
675+
char *larger = cbm_operation_json_write(probe);
676+
ASSERT_NOT_NULL(larger);
677+
ASSERT_TRUE(strlen(larger) > (size_t)tokens * CBM_OUTPUT_BYTES_PER_TOKEN);
678+
free(larger);
679+
yyjson_mut_doc_free(probe);
680+
yyjson_doc_free(doc);
681+
free(reply);
682+
}
683+
yyjson_doc_free(base_doc);
684+
free(base);
685+
PASS();
686+
}
687+
547688
TEST(integ_mcp_trace_path_output_budget) {
548689
char args[512];
549690

@@ -1203,6 +1344,8 @@ SUITE(integration) {
12031344
RUN_TEST(integ_mcp_trace_path_cross_service);
12041345
RUN_TEST(integ_mcp_search_code_match_limit_and_budget);
12051346
RUN_TEST(integ_mcp_trace_path_output_budget);
1347+
RUN_TEST(integ_mcp_snippet_line_and_output_budget);
1348+
RUN_TEST(integ_js_path_module_default_client);
12061349
RUN_TEST(integ_mcp_index_status);
12071350
RUN_TEST(integ_mcp_adr_outline_fence_status);
12081351

0 commit comments

Comments
 (0)