Skip to content

Commit 84fffef

Browse files
committed
fix: preserve CPU headroom during background indexing
Fixes #1084 Signed-off-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>
1 parent db2f104 commit 84fffef

8 files changed

Lines changed: 209 additions & 12 deletions

File tree

‎src/daemon/application.c‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1939,9 +1939,9 @@ static char *application_auto_index_args(const char *root_path) {
19391939
return NULL;
19401940
}
19411941
yyjson_mut_doc_set_root(document, root);
1942-
char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path)
1943-
? yyjson_mut_write(document, 0, NULL)
1944-
: NULL;
1942+
bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) &&
1943+
yyjson_mut_obj_add_bool(document, root, "_background", true);
1944+
char *args = encoded ? yyjson_mut_write(document, 0, NULL) : NULL;
19451945
yyjson_mut_doc_free(document);
19461946
return args;
19471947
}
@@ -3383,7 +3383,8 @@ static int application_background_index(cbm_daemon_application_t *application,
33833383
return -1;
33843384
}
33853385
yyjson_mut_doc_set_root(document, root);
3386-
bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root);
3386+
bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root) &&
3387+
yyjson_mut_obj_add_bool(document, root, "_background", true);
33873388
char *default_project = cbm_project_name_from_path(canonical_root);
33883389
bool custom_project =
33893390
project_name[0] && (!default_project || strcmp(default_project, project_name) != 0);

‎src/mcp/mcp.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8611,6 +8611,7 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p
86118611
yyjson_mut_val *root = yyjson_mut_obj(doc);
86128612
yyjson_mut_doc_set_root(doc, root);
86138613
yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path);
8614+
yyjson_mut_obj_add_bool(doc, root, "_background", true);
86148615
char *args = yy_doc_to_str(doc);
86158616
yyjson_mut_doc_free(doc);
86168617
if (!args) {
@@ -8721,6 +8722,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
87218722
char *repo_path = cbm_mcp_get_string_arg(args, "repo_path");
87228723
char *mode_str = cbm_mcp_get_string_arg(args, "mode");
87238724
char *name_override = cbm_mcp_get_string_arg(args, "name");
8725+
bool background = cbm_mcp_get_bool_arg(args, "_background");
87248726
cbm_normalize_path_sep(repo_path);
87258727

87268728
if (!repo_path) {
@@ -8868,6 +8870,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
88688870
free(repo_path);
88698871
return cbm_mcp_text_result("failed to create pipeline", true);
88708872
}
8873+
cbm_pipeline_set_background(p, background);
88718874
if (name_override && name_override[0] && !cbm_pipeline_set_project_name(p, name_override)) {
88728875
cbm_pipeline_free(p);
88738876
mcp_project_mutation_end(srv, mutation_project);
@@ -12825,6 +12828,7 @@ static void *autoindex_thread(void *arg) {
1282512828
cbm_log_warn("autoindex.err", "msg", "pipeline_create_failed");
1282612829
return NULL;
1282712830
}
12831+
cbm_pipeline_set_background(p, true);
1282812832

1282912833
/* Block until any concurrent pipeline finishes */
1283012834
cbm_pipeline_lock();

‎src/pipeline/pipeline.c‎

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,18 @@ static inline void *intptr_to_ptr(intptr_t v) {
6565
* Atomic spinlock: 0 = free, 1 = locked. */
6666
static atomic_int g_pipeline_busy = 0;
6767

68+
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
69+
static atomic_int g_last_worker_count = 0;
70+
71+
void cbm_pipeline_worker_count_test_reset(void) {
72+
atomic_store(&g_last_worker_count, 0);
73+
}
74+
75+
int cbm_pipeline_worker_count_test_last(void) {
76+
return atomic_load(&g_last_worker_count);
77+
}
78+
#endif
79+
6880
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
6981
static atomic_bool g_persist_test_fail_after_stage_dump = false;
7082
static atomic_bool g_persist_test_cancel_after_predump = false;
@@ -157,6 +169,7 @@ struct cbm_pipeline {
157169
atomic_int cancelled_storage;
158170
atomic_int *cancelled;
159171
bool persistence; /* write .codebase-memory/graph.db.zst after indexing */
172+
bool background; /* reserve CPU headroom when no user is waiting */
160173

161174
/* Indexing state (set during run) */
162175
cbm_gbuf_t *gbuf;
@@ -313,6 +326,12 @@ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled) {
313326
}
314327
}
315328

329+
void cbm_pipeline_set_background(cbm_pipeline_t *p, bool background) {
330+
if (p) {
331+
p->background = background;
332+
}
333+
}
334+
316335
bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) {
317336
if (!p || !name || !name[0]) {
318337
return false;
@@ -518,12 +537,18 @@ void cbm_pipeline_set_committed_counts(cbm_pipeline_t *p, int nodes, int edges)
518537
* crasher; a parallel re-run would race the marker. Honour that override
519538
* everywhere the worker count drives the parallel/sequential decision, so the
520539
* whole extraction phase collapses to the deterministic sequential path. */
521-
static int effective_worker_count(bool initial) {
540+
int cbm_pipeline_worker_count(const cbm_pipeline_t *p) {
522541
const char *st = getenv("CBM_INDEX_SINGLE_THREAD");
542+
int workers;
523543
if (st && st[0] == '1') {
524-
return 1;
544+
workers = 1;
545+
} else {
546+
workers = cbm_default_worker_count(!p || !p->background);
525547
}
526-
return cbm_default_worker_count(initial);
548+
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
549+
atomic_store(&g_last_worker_count, workers);
550+
#endif
551+
return workers;
527552
}
528553

529554
/* Resolve the DB path for this pipeline. Caller must free(). */
@@ -2065,7 +2090,7 @@ static int run_githistory(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) {
20652090
gh_compute_arg_t gh_arg = {.repo_path = ctx->repo_path, .result = &gh_result};
20662091

20672092
if (p->mode != CBM_MODE_FAST) {
2068-
if (effective_worker_count(true) > SKIP_ONE) {
2093+
if (cbm_pipeline_worker_count(p) > SKIP_ONE) {
20692094
if (cbm_thread_create(&gh_thread, 0, gh_compute_thread_fn, &gh_arg) == 0) {
20702095
gh_threaded = true;
20712096
}
@@ -2163,7 +2188,7 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
21632188
return CBM_NOT_FOUND;
21642189
}
21652190

2166-
int worker_count = effective_worker_count(true);
2191+
int worker_count = cbm_pipeline_worker_count(p);
21672192
CBM_PROF_START(t_extract_total);
21682193
int rc = (worker_count > SKIP_ONE && file_count > MIN_FILES_FOR_PARALLEL)
21692194
? run_parallel_pipeline(p, ctx, files, file_count, worker_count, &t)

‎src/pipeline/pipeline.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,19 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path, cbm
5353
* When enabled, the pipeline writes a compressed artifact after indexing. */
5454
void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled);
5555

56+
/* Mark work that runs without a waiting user. Background pipelines reserve
57+
* CPU headroom; foreground pipelines retain the initial-index all-core policy. */
58+
void cbm_pipeline_set_background(cbm_pipeline_t *p, bool background);
59+
60+
/* Resolve the worker policy for this pipeline, including environment and
61+
* crash-recovery overrides. */
62+
int cbm_pipeline_worker_count(const cbm_pipeline_t *p);
63+
64+
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
65+
void cbm_pipeline_worker_count_test_reset(void);
66+
int cbm_pipeline_worker_count_test_last(void);
67+
#endif
68+
5669
/* Free a pipeline and all its internal state. NULL-safe. */
5770
void cbm_pipeline_free(cbm_pipeline_t *p);
5871

‎src/pipeline/pipeline_incremental.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1211,7 +1211,7 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed
12111211
* a full build takes, which is what makes its output converge. */
12121212

12131213
#define MIN_FILES_FOR_PARALLEL_INCR 50
1214-
int worker_count = cbm_default_worker_count(true);
1214+
int worker_count = cbm_pipeline_worker_count(ctx->pipeline);
12151215
bool use_parallel =
12161216
closure != NULL || (worker_count > SKIP_ONE && ci > MIN_FILES_FOR_PARALLEL_INCR);
12171217

@@ -1580,7 +1580,7 @@ static int closure_probe_surfaces(cbm_pipeline_t *p, const char *project,
15801580
_Atomic int64_t probe_ids;
15811581
atomic_init(&probe_ids, cbm_gbuf_next_id(probe_gbuf));
15821582
rc = cbm_parallel_extract(&probe_ctx, probe_files, probe_count, cache, &probe_ids,
1583-
cbm_default_worker_count(true));
1583+
cbm_pipeline_worker_count(p));
15841584
}
15851585
if (rc == 0) {
15861586
char **def_modules = (char **)calloc((size_t)probe_count, sizeof(char *));
@@ -2048,7 +2048,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
20482048
elig[elig_count++] = row;
20492049
}
20502050
}
2051-
int workers = cbm_default_worker_count(true);
2051+
int workers = cbm_pipeline_worker_count(p);
20522052
if (workers < 1) {
20532053
workers = 1;
20542054
}

‎tests/test_daemon_application.c‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1369,6 +1369,7 @@ typedef struct {
13691369
char quarantine_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP];
13701370
char quarantine_seen[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP];
13711371
size_t memory_budgets[APP_FAKE_MAX_ATTEMPTS];
1372+
bool background_requests[APP_FAKE_MAX_ATTEMPTS];
13721373
} app_fake_worker_context_t;
13731374

13741375
typedef struct {
@@ -1433,6 +1434,8 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem
14331434
worker->result.exit_code = -1;
14341435
if (worker->attempt < APP_FAKE_MAX_ATTEMPTS) {
14351436
context->memory_budgets[worker->attempt] = memory_budget_bytes;
1437+
context->background_requests[worker->attempt] =
1438+
cbm_mcp_get_bool_arg(args_json, "_background");
14361439
if (marker_file) {
14371440
(void)snprintf(context->marker_paths[worker->attempt], APP_TEST_PATH_CAP, "%s",
14381441
marker_file);
@@ -2115,6 +2118,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) {
21152118
ASSERT_TRUE(restricted_started_nothing);
21162119
ASSERT_TRUE(first_initialized);
21172120
ASSERT_TRUE(first_owned);
2121+
ASSERT_TRUE(fake.background_requests[0]);
21182122
ASSERT_TRUE(second_initialized);
21192123
ASSERT_TRUE(coalesced);
21202124
ASSERT_TRUE(restricted_disconnect_kept_job);
@@ -3504,6 +3508,7 @@ TEST(daemon_application_request_cancel_preserves_persistent_watch_and_session) {
35043508
ASSERT_TRUE(thread_started);
35053509
ASSERT_TRUE(subscribed);
35063510
ASSERT_TRUE(worker_started);
3511+
ASSERT_FALSE(fixture.fake.background_requests[0]);
35073512
ASSERT_TRUE(request_returned);
35083513
ASSERT_TRUE(thread_joined);
35093514
ASSERT_EQ(request.status, CBM_DAEMON_RUNTIME_APPLICATION_CANCELLED);
@@ -3801,6 +3806,7 @@ TEST(daemon_application_final_cancel_drains_admitted_watcher_job) {
38013806
ASSERT_TRUE(fixture_ready);
38023807
ASSERT_TRUE(thread_started);
38033808
ASSERT_TRUE(worker_started);
3809+
ASSERT_TRUE(fixture.fake.background_requests[0]);
38043810
ASSERT_TRUE(job_active);
38053811
ASSERT_EQ(watches_after_cancel, 0);
38063812
ASSERT_TRUE(thread_joined);

‎tests/test_mcp.c‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12970,6 +12970,73 @@ TEST(autoindex_skip_reports_numeric_limit_issue1466) {
1297012970
PASS();
1297112971
}
1297212972

12973+
TEST(mcp_auto_index_in_process_uses_background_worker_policy) {
12974+
char cache[256];
12975+
char repo[512];
12976+
(void)snprintf(cache, sizeof(cache), "%s/cbm-autoindex-workers-XXXXXX", cbm_tmpdir());
12977+
bool cache_ready = cbm_mkdtemp(cache) != NULL;
12978+
(void)snprintf(repo, sizeof(repo), "%s/repo", cache);
12979+
char source[640];
12980+
(void)snprintf(source, sizeof(source), "%s/main.py", repo);
12981+
bool repo_ready = cache_ready && th_mkdir_p(repo) == 0 &&
12982+
th_write_file(source, "def background_index():\n return True\n") == 0;
12983+
12984+
mcp_test_env_backup_t environment[] = {
12985+
{.name = "CBM_CACHE_DIR"},
12986+
{.name = "CBM_WORKERS"},
12987+
{.name = "CBM_INDEX_SINGLE_THREAD"},
12988+
};
12989+
bool environment_saved = true;
12990+
for (size_t i = 0; i < sizeof(environment) / sizeof(environment[0]); i++) {
12991+
const char *value = getenv(environment[i].name);
12992+
environment[i].present = value != NULL;
12993+
environment[i].value = value ? strdup(value) : NULL;
12994+
environment_saved = environment_saved && (!value || environment[i].value);
12995+
}
12996+
bool environment_ready =
12997+
environment_saved && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0 &&
12998+
cbm_unsetenv("CBM_WORKERS") == 0 && cbm_unsetenv("CBM_INDEX_SINGLE_THREAD") == 0;
12999+
13000+
char old_cwd[CBM_SZ_4K] = {0};
13001+
bool cwd_ready = repo_ready && environment_ready && cbm_getcwd(old_cwd, sizeof(old_cwd)) &&
13002+
cbm_chdir(repo) == 0;
13003+
cbm_config_t *config = cwd_ready ? cbm_config_open(cache) : NULL;
13004+
bool config_ready = config && cbm_config_set(config, CBM_CONFIG_AUTO_INDEX, "true") == 0 &&
13005+
cbm_config_set(config, CBM_CONFIG_AUTO_WATCH, "false") == 0;
13006+
cbm_pipeline_worker_count_test_reset();
13007+
cbm_mcp_server_t *server = config_ready ? cbm_mcp_server_new(NULL) : NULL;
13008+
char *response = NULL;
13009+
if (server) {
13010+
cbm_mcp_server_set_config(server, config);
13011+
response = cbm_mcp_server_handle(
13012+
server, "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\",\"params\":{}}");
13013+
cbm_mcp_server_free(server); /* joins the automatic index thread */
13014+
}
13015+
int selected_workers = cbm_pipeline_worker_count_test_last();
13016+
bool server_ready = server != NULL;
13017+
bool response_ready = response != NULL;
13018+
13019+
free(response);
13020+
cbm_config_close(config);
13021+
if (cwd_ready) {
13022+
(void)cbm_chdir(old_cwd);
13023+
}
13024+
mcp_test_restore_env(environment, sizeof(environment) / sizeof(environment[0]));
13025+
bool cleaned = !cache_ready || th_rmtree(cache) == 0;
13026+
13027+
ASSERT_TRUE(cache_ready);
13028+
ASSERT_TRUE(repo_ready);
13029+
ASSERT_TRUE(environment_saved);
13030+
ASSERT_TRUE(environment_ready);
13031+
ASSERT_TRUE(cwd_ready);
13032+
ASSERT_TRUE(config_ready);
13033+
ASSERT_TRUE(server_ready);
13034+
ASSERT_TRUE(response_ready);
13035+
ASSERT_EQ(selected_workers, cbm_default_worker_count(false));
13036+
ASSERT_TRUE(cleaned);
13037+
PASS();
13038+
}
13039+
1297313040
/* ══════════════════════════════════════════════════════════════════
1297413041
* #853 — auto_watch=false must ALSO gate the SUPERVISED fresh-index
1297513042
* watcher registration (keystone × #849 merge interaction)
@@ -14186,6 +14253,7 @@ SUITE(mcp) {
1418614253
RUN_TEST(mcp_auto_watch_false_skips_watcher_on_connect);
1418714254
RUN_TEST(mcp_auto_watch_false_skips_supervised_autoindex_issue853);
1418814255
RUN_TEST(autoindex_skip_reports_numeric_limit_issue1466);
14256+
RUN_TEST(mcp_auto_index_in_process_uses_background_worker_policy);
1418914257
}
1419014258

1419114259
/* Kept separate so daemon-coordination regressions can be iterated without

‎tests/test_pipeline.c‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,84 @@ TEST(pipeline_create_free) {
116116
PASS();
117117
}
118118

119+
TEST(pipeline_background_worker_policy_preserves_overrides) {
120+
const char *old_workers = getenv("CBM_WORKERS");
121+
char *saved_workers = old_workers ? strdup(old_workers) : NULL;
122+
const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD");
123+
char *saved_single = old_single ? strdup(old_single) : NULL;
124+
cbm_unsetenv("CBM_WORKERS");
125+
cbm_unsetenv("CBM_INDEX_SINGLE_THREAD");
126+
127+
cbm_pipeline_t *foreground = cbm_pipeline_new("/some/path", NULL, CBM_MODE_FULL);
128+
cbm_pipeline_t *background = cbm_pipeline_new("/some/path", NULL, CBM_MODE_FULL);
129+
ASSERT_NOT_NULL(foreground);
130+
ASSERT_NOT_NULL(background);
131+
cbm_pipeline_set_background(background, true);
132+
133+
int foreground_default = cbm_pipeline_worker_count(foreground);
134+
int background_default = cbm_pipeline_worker_count(background);
135+
bool defaults_match = foreground_default == cbm_default_worker_count(true) &&
136+
background_default == cbm_default_worker_count(false) &&
137+
foreground_default >= 1 && background_default >= 1;
138+
139+
cbm_setenv("CBM_WORKERS", "3", 1);
140+
bool env_override = cbm_pipeline_worker_count(foreground) == 3 &&
141+
cbm_pipeline_worker_count(background) == 3;
142+
143+
cbm_setenv("CBM_INDEX_SINGLE_THREAD", "1", 1);
144+
bool recovery_override = cbm_pipeline_worker_count(foreground) == 1 &&
145+
cbm_pipeline_worker_count(background) == 1;
146+
147+
cbm_pipeline_free(foreground);
148+
cbm_pipeline_free(background);
149+
saved_workers ? cbm_setenv("CBM_WORKERS", saved_workers, 1) : cbm_unsetenv("CBM_WORKERS");
150+
saved_single ? cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1)
151+
: cbm_unsetenv("CBM_INDEX_SINGLE_THREAD");
152+
free(saved_workers);
153+
free(saved_single);
154+
155+
ASSERT_TRUE(defaults_match);
156+
ASSERT_TRUE(env_override);
157+
ASSERT_TRUE(recovery_override);
158+
PASS();
159+
}
160+
161+
TEST(pipeline_background_policy_preserves_index_results) {
162+
ASSERT_EQ(setup_test_repo(), 0);
163+
char foreground_db[512];
164+
char background_db[512];
165+
(void)snprintf(foreground_db, sizeof(foreground_db), "%s/foreground.db", g_tmpdir);
166+
(void)snprintf(background_db, sizeof(background_db), "%s/background.db", g_tmpdir);
167+
168+
cbm_pipeline_t *foreground =
169+
cbm_pipeline_new(g_tmpdir, foreground_db, CBM_MODE_FULL);
170+
cbm_pipeline_t *background =
171+
cbm_pipeline_new(g_tmpdir, background_db, CBM_MODE_FULL);
172+
ASSERT_NOT_NULL(foreground);
173+
ASSERT_NOT_NULL(background);
174+
cbm_pipeline_set_background(background, true);
175+
176+
int foreground_rc = cbm_pipeline_run(foreground);
177+
int background_rc = cbm_pipeline_run(background);
178+
int foreground_nodes = -1;
179+
int foreground_edges = -1;
180+
int background_nodes = -1;
181+
int background_edges = -1;
182+
cbm_pipeline_get_committed_counts(foreground, &foreground_nodes, &foreground_edges);
183+
cbm_pipeline_get_committed_counts(background, &background_nodes, &background_edges);
184+
185+
cbm_pipeline_free(foreground);
186+
cbm_pipeline_free(background);
187+
teardown_test_repo();
188+
189+
ASSERT_EQ(foreground_rc, 0);
190+
ASSERT_EQ(background_rc, 0);
191+
ASSERT_GT(foreground_nodes, 0);
192+
ASSERT_EQ(background_nodes, foreground_nodes);
193+
ASSERT_EQ(background_edges, foreground_edges);
194+
PASS();
195+
}
196+
119197
TEST(pipeline_null_repo) {
120198
cbm_pipeline_t *p = cbm_pipeline_new(NULL, NULL, CBM_MODE_FULL);
121199
ASSERT_NULL(p);
@@ -13090,6 +13168,8 @@ SUITE(pipeline) {
1309013168
RUN_TEST(pipeline_lock_release_allows_contender);
1309113169
/* Lifecycle */
1309213170
RUN_TEST(pipeline_create_free);
13171+
RUN_TEST(pipeline_background_worker_policy_preserves_overrides);
13172+
RUN_TEST(pipeline_background_policy_preserves_index_results);
1309313173
RUN_TEST(pipeline_null_repo);
1309413174
RUN_TEST(pipeline_free_null);
1309513175
RUN_TEST(pipeline_cancel);

0 commit comments

Comments
 (0)