From 358d6e76ab749a7808ce02089b1b5b8d6431fc82 Mon Sep 17 00:00:00 2001 From: Pxx500 Date: Thu, 20 Aug 2026 16:29:19 +0200 Subject: [PATCH 01/27] fix(mcp): recover unsent requests after daemon loss Signed-off-by: Pxx500 --- src/daemon/application.c | 27 +-- src/daemon/application.h | 5 +- src/daemon/frontend.c | 183 +++++++++++++++----- src/daemon/frontend.h | 22 ++- src/daemon/runtime.c | 14 +- src/daemon/runtime.h | 5 +- src/main.c | 60 +++++-- tests/test_daemon_frontend.c | 322 +++++++++++++++++++++++++++++++---- tests/test_daemon_runtime.c | 27 ++- 9 files changed, 547 insertions(+), 118 deletions(-) diff --git a/src/daemon/application.c b/src/daemon/application.c index 69ff52ec7f..58fb6bb311 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -3116,7 +3116,7 @@ cbm_daemon_runtime_application_callbacks_t cbm_daemon_application_runtime_callba static cbm_daemon_runtime_application_status_t application_client_exchange_tagged( cbm_daemon_runtime_client_t *client, cbm_daemon_runtime_application_token_t request_token, uint8_t *request, uint32_t request_length, uint8_t **response_out, - uint32_t *response_length_out, uint32_t timeout_ms) { + uint32_t *response_length_out, bool *request_sent_out, uint32_t timeout_ms) { uint8_t *response = NULL; uint32_t response_length = 0; if (response_out) { @@ -3125,13 +3125,16 @@ static cbm_daemon_runtime_application_status_t application_client_exchange_tagge if (response_length_out) { *response_length_out = 0; } + if (request_sent_out) { + *request_sent_out = false; + } cbm_daemon_runtime_application_status_t status = request_token == CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID ? cbm_daemon_runtime_client_application_request(client, request, request_length, &response, &response_length, timeout_ms) - : cbm_daemon_runtime_client_application_request_tagged(client, request_token, request, - request_length, &response, - &response_length, timeout_ms); + : cbm_daemon_runtime_client_application_request_tagged( + client, request_token, request, request_length, &response, &response_length, + request_sent_out, timeout_ms); free(request); if (status != CBM_DAEMON_RUNTIME_APPLICATION_OK) { free(response); @@ -3162,7 +3165,7 @@ static cbm_daemon_runtime_application_status_t application_client_exchange( uint8_t **response_out, uint32_t *response_length_out, uint32_t timeout_ms) { return application_client_exchange_tagged(client, CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID, request, request_length, response_out, - response_length_out, timeout_ms); + response_length_out, NULL, timeout_ms); } cbm_daemon_runtime_application_status_t cbm_daemon_application_client_set_context( @@ -3290,7 +3293,7 @@ cbm_daemon_runtime_application_status_t cbm_daemon_application_client_ui_readine static cbm_daemon_runtime_application_status_t application_client_text_request_tagged( cbm_daemon_runtime_client_t *client, cbm_daemon_runtime_application_token_t request_token, cbm_daemon_application_request_kind_t kind, const char *text, uint8_t **response_out, - uint32_t *response_length_out, uint32_t timeout_ms) { + uint32_t *response_length_out, bool *request_sent_out, uint32_t timeout_ms) { if (!client || !text || !text[0]) { return CBM_DAEMON_RUNTIME_APPLICATION_REJECTED; } @@ -3306,7 +3309,7 @@ static cbm_daemon_runtime_application_status_t application_client_text_request_t memcpy(request + 1, text, text_length); return application_client_exchange_tagged(client, request_token, request, (uint32_t)text_length + 1U, response_out, - response_length_out, timeout_ms); + response_length_out, request_sent_out, timeout_ms); } static cbm_daemon_runtime_application_status_t application_client_text_request( @@ -3314,7 +3317,7 @@ static cbm_daemon_runtime_application_status_t application_client_text_request( const char *text, uint8_t **response_out, uint32_t *response_length_out, uint32_t timeout_ms) { return application_client_text_request_tagged( client, CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID, kind, text, response_out, - response_length_out, timeout_ms); + response_length_out, NULL, timeout_ms); } cbm_daemon_runtime_application_status_t cbm_daemon_application_client_mcp( @@ -3327,10 +3330,10 @@ cbm_daemon_runtime_application_status_t cbm_daemon_application_client_mcp( cbm_daemon_runtime_application_status_t cbm_daemon_application_client_mcp_tagged( cbm_daemon_runtime_client_t *client, cbm_daemon_runtime_application_token_t request_token, const char *message, uint8_t **response_out, uint32_t *response_length_out, - uint32_t timeout_ms) { - return application_client_text_request_tagged(client, request_token, - CBM_DAEMON_APPLICATION_REQUEST_MCP, message, - response_out, response_length_out, timeout_ms); + bool *request_sent_out, uint32_t timeout_ms) { + return application_client_text_request_tagged( + client, request_token, CBM_DAEMON_APPLICATION_REQUEST_MCP, message, response_out, + response_length_out, request_sent_out, timeout_ms); } cbm_daemon_runtime_application_status_t cbm_daemon_application_client_tool( diff --git a/src/daemon/application.h b/src/daemon/application.h index 933186aa73..b071ea03cc 100644 --- a/src/daemon/application.h +++ b/src/daemon/application.h @@ -149,11 +149,12 @@ cbm_daemon_runtime_application_status_t cbm_daemon_application_client_mcp( cbm_daemon_runtime_client_t *client, const char *message, uint8_t **response_out, uint32_t *response_length_out, uint32_t timeout_ms); -/* Cancellable frontend variant using a token reserved on the runtime client. */ +/* Cancellable frontend variant using a token reserved on the runtime client. + * request_sent_out preserves the runtime's application-frame delivery fact. */ cbm_daemon_runtime_application_status_t cbm_daemon_application_client_mcp_tagged( cbm_daemon_runtime_client_t *client, cbm_daemon_runtime_application_token_t request_token, const char *message, uint8_t **response_out, uint32_t *response_length_out, - uint32_t timeout_ms); + bool *request_sent_out, uint32_t timeout_ms); cbm_daemon_runtime_application_status_t cbm_daemon_application_client_tool( cbm_daemon_runtime_client_t *client, const char *tool_name, const char *args_json, diff --git a/src/daemon/frontend.c b/src/daemon/frontend.c index 2dd9869011..92b0326867 100644 --- a/src/daemon/frontend.c +++ b/src/daemon/frontend.c @@ -76,6 +76,7 @@ typedef struct { cbm_mutex_t mutex; cbm_daemon_runtime_client_t *client; cbm_version_cohort_manager_t *cohort_manager; + const cbm_daemon_frontend_session_config_t *session; FILE *out; frontend_item_t queue[FRONTEND_QUEUE_CAPACITY]; size_t head; @@ -89,6 +90,7 @@ typedef struct { int64_t active_id; const char *active_id_str; cbm_daemon_runtime_application_token_t active_request_token; + bool active_cancelled; bool failed; /* Monotonic count of fully processed queue items (responses written or * cancellations acknowledged). The EOF drain below watches it to tell a @@ -352,6 +354,7 @@ static bool frontend_pop_begin(frontend_state_t *state, frontend_item_t *item) { state->active_id_str = item->id_str; item->request_token = request_token; state->active_request_token = request_token; + state->active_cancelled = false; popped = true; } cbm_mutex_unlock(&state->mutex); @@ -365,6 +368,7 @@ static void frontend_end_request(frontend_state_t *state, bool failed) { state->active_id = 0; state->active_id_str = NULL; state->active_request_token = CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID; + state->active_cancelled = false; state->failed = state->failed || failed; cbm_mutex_unlock(&state->mutex); } @@ -446,12 +450,8 @@ typedef enum { * request routes its exact runtime token without closing the authenticated * session. A queued request is marked and receives a cancellation error * without ever reaching the daemon. Stale/invalid targets are ignored. */ -static frontend_cancellation_route_t frontend_route_cancellation( - frontend_state_t *state, const char *message, - cbm_daemon_runtime_application_token_t *request_token_out) { - if (request_token_out) { - *request_token_out = CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID; - } +static frontend_cancellation_route_t frontend_route_cancellation(frontend_state_t *state, + const char *message) { cbm_jsonrpc_request_t request = {0}; if (!frontend_parse_cancellation(message, &request)) { return FRONTEND_CANCELLATION_NONE; @@ -460,12 +460,9 @@ static frontend_cancellation_route_t frontend_route_cancellation( frontend_cancellation_route_t route = FRONTEND_CANCELLATION_STALE; cbm_mutex_lock(&state->mutex); if (state->in_request && state->active_has_id && - state->active_request_token != CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID && cbm_mcp_cancel_request_matches(request.params_raw, state->active_id, state->active_id_str)) { - if (request_token_out) { - *request_token_out = state->active_request_token; - } + state->active_cancelled = true; route = FRONTEND_CANCELLATION_ACTIVE; } else { for (size_t offset = 0; offset < state->count; offset++) { @@ -484,6 +481,89 @@ static frontend_cancellation_route_t frontend_route_cancellation( return route; } +static cbm_daemon_runtime_cancel_result_t frontend_cancel_active(frontend_state_t *state) { + cbm_mutex_lock(&state->mutex); + cbm_daemon_runtime_cancel_result_t result = CBM_DAEMON_RUNTIME_CANCEL_STALE; + if (state->in_request && state->active_cancelled && state->client && + state->active_request_token != CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID) { + result = cbm_daemon_runtime_client_application_cancel(state->client, + state->active_request_token); + } else if (state->in_request && state->active_cancelled) { + /* A reconnect has no daemon request to cancel yet. The worker observes + * active_cancelled before retrying the frame. */ + result = CBM_DAEMON_RUNTIME_CANCEL_ACCEPTED; + } + cbm_mutex_unlock(&state->mutex); + return result; +} + +static bool frontend_configure_client(cbm_daemon_runtime_client_t *client, + const cbm_daemon_frontend_session_config_t *session) { + uint32_t timeout_ms = session->bootstrap.connect_timeout_ms; + if (cbm_daemon_application_client_set_context( + client, session->session_root, session->allowed_root, session->tool_profile, NULL, NULL, + timeout_ms) != CBM_DAEMON_RUNTIME_APPLICATION_OK) { + return false; + } + return session->ui_update_mask == 0 || + cbm_daemon_application_client_set_ui_config( + client, session->ui_update_mask, session->ui_enabled, session->ui_port, + timeout_ms) == CBM_DAEMON_RUNTIME_APPLICATION_OK; +} + +static bool frontend_recover_client(frontend_state_t *state, frontend_item_t *item, + bool *cancelled_out) { + if (cancelled_out) { + *cancelled_out = false; + } + + cbm_mutex_lock(&state->mutex); + if (state->stopping || !state->client) { + cbm_mutex_unlock(&state->mutex); + return false; + } + cbm_daemon_runtime_client_t *previous = state->client; + state->client = NULL; + state->active_request_token = CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID; + cbm_mutex_unlock(&state->mutex); + if (previous) { + (void)cbm_daemon_runtime_client_close(previous, FRONTEND_CLOSE_TIMEOUT_MS); + } + + cbm_daemon_bootstrap_result_t bootstrap = {0}; + cbm_daemon_bootstrap_status_t bootstrap_status = + cbm_daemon_bootstrap_execute(&state->session->bootstrap, &bootstrap); + cbm_daemon_runtime_client_t *replacement = + bootstrap_status == CBM_DAEMON_BOOTSTRAP_CONNECTED ? bootstrap.client : NULL; + bool configured = replacement && frontend_configure_client(replacement, state->session); + cbm_daemon_runtime_application_token_t replacement_token = + CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID; + + cbm_mutex_lock(&state->mutex); + bool cancelled = state->active_cancelled; + bool usable = configured && !state->stopping && !state->failed; + if (usable && !cancelled) { + usable = + cbm_daemon_runtime_client_application_token_reserve(replacement, &replacement_token); + } + if (usable) { + state->client = replacement; + if (!cancelled) { + state->active_request_token = replacement_token; + item->request_token = replacement_token; + } + if (cancelled_out) { + *cancelled_out = cancelled; + } + } + cbm_mutex_unlock(&state->mutex); + + if (!usable && replacement) { + (void)cbm_daemon_runtime_client_close(replacement, FRONTEND_CLOSE_TIMEOUT_MS); + } + return usable; +} + static void *frontend_worker(void *opaque) { frontend_state_t *state = opaque; for (;;) { @@ -504,10 +584,27 @@ static void *frontend_worker(void *opaque) { if (item.cancelled) { failed = !frontend_write_cancelled_response(state->out, &item); } else { + bool request_sent = false; cbm_daemon_runtime_application_status_t status = - cbm_daemon_application_client_mcp_tagged(state->client, item.request_token, - item.message, &response, &response_length, - FRONTEND_REQUEST_TIMEOUT_MS); + cbm_daemon_application_client_mcp_tagged( + state->client, item.request_token, item.message, &response, &response_length, + &request_sent, FRONTEND_REQUEST_TIMEOUT_MS); + if (status == CBM_DAEMON_RUNTIME_APPLICATION_TRANSPORT_ERROR && !request_sent) { + free(response); + response = NULL; + response_length = 0; + bool cancelled = false; + if (frontend_recover_client(state, &item, &cancelled)) { + if (cancelled) { + status = CBM_DAEMON_RUNTIME_APPLICATION_CANCELLED; + } else { + bool retry_sent = false; + status = cbm_daemon_application_client_mcp_tagged( + state->client, item.request_token, item.message, &response, + &response_length, &retry_sent, FRONTEND_REQUEST_TIMEOUT_MS); + } + } + } if (status == CBM_DAEMON_RUNTIME_APPLICATION_CANCELLED) { failed = !frontend_write_cancelled_response(state->out, &item); } else { @@ -526,12 +623,11 @@ static void *frontend_worker(void *opaque) { atomic_fetch_add_explicit(&state->completed_items, 1, memory_order_release); if (failed) { if (!expected_stop) { - /* A failed daemon transport cannot wake a thread blocked in - * stdio portably. This frontend owns no state: immediate - * process exit closes the kernel IPC handle, which cancels - * daemon session ownership without a detached reader or an - * unsafe cross-thread fclose. Logging or flushing here could - * itself block on agent-owned stdout/stderr. */ + /* Recovery was unsafe or the one safe retry failed. Immediate + * process exit wakes an agent blocked on stdio and closes the + * kernel IPC handle without a detached reader or cross-thread + * fclose. Logging or flushing here could itself block on + * agent-owned stdout/stderr. */ _Exit(EXIT_FAILURE); } break; @@ -614,11 +710,11 @@ static bool frontend_enqueue(frontend_state_t *state, char *message, bool conten static bool frontend_stop_begin(frontend_state_t *state) { cbm_mutex_lock(&state->mutex); state->stopping = true; + cbm_daemon_runtime_client_t *client = state->client; + bool stop_begun = !client || cbm_daemon_runtime_client_close_begin(client); cbm_mutex_unlock(&state->mutex); - /* Retain the client allocation until the worker is joined. This covers the - * boundary where the worker has claimed an item but has not yet entered the - * runtime exchange: a late call observes closing instead of freed memory. */ - return cbm_daemon_runtime_client_close_begin(state->client); + /* NULL means recovery owns cleanup. Otherwise retain the client until join. */ + return stop_begun; } static bool frontend_cancel_for_maintenance(void *opaque) { @@ -628,24 +724,33 @@ static bool frontend_cancel_for_maintenance(void *opaque) { cbm_mutex_lock(&state->mutex); state->stopping = true; if (state->in_request) { + state->active_cancelled = true; request_token = state->active_request_token; } - cbm_mutex_unlock(&state->mutex); if (request_token == CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID) { + cbm_mutex_unlock(&state->mutex); return false; } - return cbm_daemon_runtime_client_application_cancel(state->client, request_token) == - CBM_DAEMON_RUNTIME_CANCEL_ACCEPTED; + bool cancelled = state->client && + cbm_daemon_runtime_client_application_cancel(state->client, request_token) == + CBM_DAEMON_RUNTIME_CANCEL_ACCEPTED; + cbm_mutex_unlock(&state->mutex); + return cancelled; } int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client, - cbm_version_cohort_manager_t *cohort_manager, FILE *in, FILE *out) { - if (!client || !cohort_manager || !in || !out) { + cbm_version_cohort_manager_t *cohort_manager, + const cbm_daemon_frontend_session_config_t *session, FILE *in, + FILE *out) { + if (!client || !cohort_manager || !session || !session->bootstrap.endpoint || + !session->bootstrap.identity || !session->session_root || !session->session_root[0] || + !in || !out) { return -1; } frontend_state_t state = { .client = client, .cohort_manager = cohort_manager, + .session = session, .out = out, }; cbm_mutex_init(&state.mutex); @@ -669,7 +774,7 @@ int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client, } int result = 0; - bool close_begun = false; + bool stop_begun = false; bool clean_eof = false; for (;;) { char *message = NULL; @@ -681,18 +786,14 @@ int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client, free(message); break; } - cbm_daemon_runtime_application_token_t cancel_token = - CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID; - frontend_cancellation_route_t cancellation = - frontend_route_cancellation(&state, message, &cancel_token); + frontend_cancellation_route_t cancellation = frontend_route_cancellation(&state, message); if (cancellation != FRONTEND_CANCELLATION_NONE) { free(message); if (cancellation == FRONTEND_CANCELLATION_ACTIVE) { - cbm_daemon_runtime_cancel_result_t cancelled = - cbm_daemon_runtime_client_application_cancel(state.client, cancel_token); + cbm_daemon_runtime_cancel_result_t cancelled = frontend_cancel_active(&state); if (cancelled == CBM_DAEMON_RUNTIME_CANCEL_ERROR) { result = -1; - close_begun = frontend_stop_begin(&state); + stop_begun = frontend_stop_begin(&state); break; } } @@ -707,7 +808,7 @@ int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client, } } - if (clean_eof && !close_begun) { + if (clean_eof && !stop_begun) { frontend_input_closed(&state); /* EOF ends INPUT, not accepted work: as long as queued items keep * completing, every already-enqueued request still receives its @@ -733,8 +834,8 @@ int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client, } } } - if (!close_begun) { - close_begun = frontend_stop_begin(&state); + if (!stop_begun) { + stop_begun = frontend_stop_begin(&state); } frontend_join_watchdog_t watchdog; cbm_thread_t watchdog_thread; @@ -759,9 +860,9 @@ int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client, if (!cbm_daemon_maintenance_monitor_stop(&maintenance_monitor)) { _Exit(EXIT_FAILURE); } - if (close_begun) { + if (stop_begun && state.client) { (void)cbm_daemon_runtime_client_close_finish(state.client, FRONTEND_CLOSE_TIMEOUT_MS); - } else { + } else if (!stop_begun) { result = -1; } for (size_t i = 0; i < FRONTEND_QUEUE_CAPACITY; i++) { diff --git a/src/daemon/frontend.h b/src/daemon/frontend.h index f23395774a..10255687fd 100644 --- a/src/daemon/frontend.h +++ b/src/daemon/frontend.h @@ -4,14 +4,29 @@ #ifndef CBM_DAEMON_FRONTEND_H #define CBM_DAEMON_FRONTEND_H +#include "daemon/bootstrap.h" #include "daemon/runtime.h" #include "daemon/version_cohort.h" +#include "mcp/mcp.h" #include #include typedef struct cbm_daemon_maintenance_monitor cbm_daemon_maintenance_monitor_t; +/* Everything the stdio frontend must replay when its authenticated daemon + * connection is replaced. Pointer fields are borrowed for the complete + * frontend call. */ +typedef struct { + cbm_daemon_bootstrap_config_t bootstrap; + const char *session_root; + const char *allowed_root; + cbm_mcp_tool_profile_t tool_profile; + uint8_t ui_update_mask; + bool ui_enabled; + int ui_port; +} cbm_daemon_frontend_session_config_t; + /* Called once when install/update/uninstall requests an active local command * to stop cooperatively. Returning false does not authorize the command to * outlive the bounded grace period. */ @@ -60,9 +75,12 @@ uint64_t cbm_daemon_frontend_test_worker_idle_cycles(void); * either thread is blocked in stdio, requests cooperative cancellation for the * exact active request, and then bounds process exit. Kernel IPC close cancels * only this session's daemon work. EOF/parse failure closes the authenticated - * session. An unexpected daemon transport failure likewise terminates the + * session. A transport failure before the application frame is sent replaces + * the daemon client and retries once. Sent or repeated failures terminate the * process so an agent waiting with stdin still open observes server EOF. */ int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client, - cbm_version_cohort_manager_t *cohort_manager, FILE *in, FILE *out); + cbm_version_cohort_manager_t *cohort_manager, + const cbm_daemon_frontend_session_config_t *session, FILE *in, + FILE *out); #endif /* CBM_DAEMON_FRONTEND_H */ diff --git a/src/daemon/runtime.c b/src/daemon/runtime.c index 7db0f0225a..c9eaeda499 100644 --- a/src/daemon/runtime.c +++ b/src/daemon/runtime.c @@ -3019,13 +3019,16 @@ cbm_daemon_runtime_cancel_result_t cbm_daemon_runtime_client_application_cancel( cbm_daemon_runtime_application_status_t cbm_daemon_runtime_client_application_request_tagged( cbm_daemon_runtime_client_t *client, cbm_daemon_runtime_application_token_t request_token, const void *request, uint32_t request_length, uint8_t **response_out, - uint32_t *response_length_out, uint32_t timeout_ms) { + uint32_t *response_length_out, bool *request_sent_out, uint32_t timeout_ms) { if (response_out) { *response_out = NULL; } if (response_length_out) { *response_length_out = 0; } + if (request_sent_out) { + *request_sent_out = false; + } if (!client || !response_out || !response_length_out || request_token == CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID || timeout_ms == CBM_DAEMON_IPC_WAIT_FOREVER || @@ -3081,6 +3084,9 @@ cbm_daemon_runtime_application_status_t cbm_daemon_runtime_client_application_re request_can_send && cbm_daemon_ipc_send_frame(connection, CBM_DAEMON_FRAME_REQUEST, CBM_DAEMON_RUNTIME_OP_APPLICATION_REQUEST, wire, (uint32_t)wire_length); + if (request_sent_out) { + *request_sent_out = sent; + } bool send_cancel = false; cbm_mutex_lock(&client->state_mutex); client->application_request_sent = sent; @@ -3179,9 +3185,9 @@ cbm_daemon_runtime_application_status_t cbm_daemon_runtime_client_application_re if (!cbm_daemon_runtime_client_application_token_reserve(client, &request_token)) { return CBM_DAEMON_RUNTIME_APPLICATION_TRANSPORT_ERROR; } - return cbm_daemon_runtime_client_application_request_tagged(client, request_token, request, - request_length, response_out, - response_length_out, timeout_ms); + return cbm_daemon_runtime_client_application_request_tagged( + client, request_token, request, request_length, response_out, response_length_out, NULL, + timeout_ms); } bool cbm_daemon_runtime_client_close_begin(cbm_daemon_runtime_client_t *client) { diff --git a/src/daemon/runtime.h b/src/daemon/runtime.h index 4d53da9a4a..15521ac83d 100644 --- a/src/daemon/runtime.h +++ b/src/daemon/runtime.h @@ -399,11 +399,12 @@ cbm_daemon_runtime_application_status_t cbm_daemon_runtime_client_application_re /* Execute using the sole outstanding token returned by token_reserve(). This * is the cancellable frontend path; the legacy helper above reserves - * internally. */ + * internally. request_sent_out reports whether the application frame crossed + * the local transport, including failures while awaiting its response. */ cbm_daemon_runtime_application_status_t cbm_daemon_runtime_client_application_request_tagged( cbm_daemon_runtime_client_t *client, cbm_daemon_runtime_application_token_t request_token, const void *request, uint32_t request_length, uint8_t **response_out, - uint32_t *response_length_out, uint32_t timeout_ms); + uint32_t *response_length_out, bool *request_sent_out, uint32_t timeout_ms); /* Begin a two-phase close without freeing the client. This atomically rejects * future exchanges and interrupts an exchange already in flight. Exactly one diff --git a/src/main.c b/src/main.c index 8a575c44f2..98db00561a 100644 --- a/src/main.c +++ b/src/main.c @@ -3073,38 +3073,54 @@ int main(int argc, char **argv) { cbm_daemon_ipc_endpoint_free(endpoint); if (bootstrap_status != CBM_DAEMON_BOOTSTRAP_CONNECTED || !bootstrap_result.client) { main_report_client_bootstrap_failure(role, &bootstrap_result); + cbm_daemon_ipc_endpoint_free(endpoint); (void)main_version_cohort_close(&client_cohort_lease, &client_cohort_manager); return EXIT_FAILURE; } g_daemon_client = bootstrap_result.client; - if (role == CBM_DAEMON_PROCESS_MCP_CLIENT && - !main_set_client_context(g_daemon_client, NULL, tool_profile, NULL, NULL, - MAIN_CONNECT_TIMEOUT_MS)) { - (void)fprintf(stderr, "codebase-memory-mcp: daemon session context was rejected\n"); - (void)cbm_daemon_runtime_client_close(g_daemon_client, MAIN_CLOSE_TIMEOUT_MS); - g_daemon_client = NULL; - (void)main_version_cohort_close(&client_cohort_lease, &client_cohort_manager); - return EXIT_FAILURE; + char mcp_session_root[MAIN_PATH_CAP] = {0}; + char mcp_allowed_root[MAIN_PATH_CAP] = {0}; + const char *mcp_allowed_root_ptr = NULL; + if (role == CBM_DAEMON_PROCESS_MCP_CLIENT) { + bool context_resolved = + main_session_context(NULL, mcp_session_root, mcp_allowed_root, &mcp_allowed_root_ptr); + cbm_daemon_runtime_application_status_t context_status = + context_resolved + ? cbm_daemon_application_client_set_context(g_daemon_client, mcp_session_root, + mcp_allowed_root_ptr, tool_profile, + NULL, NULL, MAIN_CONNECT_TIMEOUT_MS) + : CBM_DAEMON_RUNTIME_APPLICATION_REJECTED; + if (context_status != CBM_DAEMON_RUNTIME_APPLICATION_OK) { + (void)fprintf(stderr, "codebase-memory-mcp: daemon session context was rejected\n"); + (void)cbm_daemon_runtime_client_close(g_daemon_client, MAIN_CLOSE_TIMEOUT_MS); + g_daemon_client = NULL; + cbm_daemon_ipc_endpoint_free(endpoint); + (void)main_version_cohort_close(&client_cohort_lease, &client_cohort_manager); + return EXIT_FAILURE; + } } /* Persist UI mutations only after the exact-build HELLO succeeds. A * conflicting binary must be observationally read-only: applying its * flags before bootstrap could reconfigure the already-running daemon * even though that client was then rejected. */ + bool mcp_ui_enabled = false; + int mcp_ui_port = 0; + uint8_t mcp_ui_update_mask = 0; if (role == CBM_DAEMON_PROCESS_MCP_CLIENT && cbm_mcp_tool_profile_allows_http(tool_profile)) { - bool ui_enabled = false; - int ui_port = 0; bool explicitly_enabled = false; - uint8_t update_mask = - parse_ui_flags(argc, argv, &ui_enabled, &ui_port, &explicitly_enabled); - if (update_mask != 0 && cbm_daemon_application_client_set_ui_config( - g_daemon_client, update_mask, ui_enabled, ui_port, - MAIN_CONNECT_TIMEOUT_MS) != CBM_DAEMON_RUNTIME_APPLICATION_OK) { + mcp_ui_update_mask = + parse_ui_flags(argc, argv, &mcp_ui_enabled, &mcp_ui_port, &explicitly_enabled); + if (mcp_ui_update_mask != 0 && + cbm_daemon_application_client_set_ui_config( + g_daemon_client, mcp_ui_update_mask, mcp_ui_enabled, mcp_ui_port, + MAIN_CONNECT_TIMEOUT_MS) != CBM_DAEMON_RUNTIME_APPLICATION_OK) { (void)fprintf(stderr, "codebase-memory-mcp: daemon UI configuration update failed\n"); (void)cbm_daemon_runtime_client_close(g_daemon_client, MAIN_CLOSE_TIMEOUT_MS); g_daemon_client = NULL; + cbm_daemon_ipc_endpoint_free(endpoint); (void)main_version_cohort_close(&client_cohort_lease, &client_cohort_manager); return EXIT_FAILURE; } @@ -3119,14 +3135,26 @@ int main(int argc, char **argv) { (void)fprintf(stderr, "codebase-memory-mcp: parent-death watchdog could not start\n"); (void)cbm_daemon_runtime_client_close(g_daemon_client, MAIN_CLOSE_TIMEOUT_MS); g_daemon_client = NULL; + cbm_daemon_ipc_endpoint_free(endpoint); (void)main_version_cohort_close(&client_cohort_lease, &client_cohort_manager); return EXIT_FAILURE; } #endif setup_signal_handlers(); - int result = cbm_daemon_frontend_mcp_run(g_daemon_client, client_cohort_manager, stdin, stdout); + cbm_daemon_frontend_session_config_t frontend_session = { + .bootstrap = bootstrap_config, + .session_root = mcp_session_root, + .allowed_root = mcp_allowed_root_ptr, + .tool_profile = tool_profile, + .ui_update_mask = mcp_ui_update_mask, + .ui_enabled = mcp_ui_enabled, + .ui_port = mcp_ui_port, + }; + int result = cbm_daemon_frontend_mcp_run(g_daemon_client, client_cohort_manager, + &frontend_session, stdin, stdout); g_daemon_client = NULL; /* frontend consumed the handle */ + cbm_daemon_ipc_endpoint_free(endpoint); bool client_cohort_cleanup = main_version_cohort_close(&client_cohort_lease, &client_cohort_manager); atomic_store(&g_shutdown, 1); diff --git a/tests/test_daemon_frontend.c b/tests/test_daemon_frontend.c index ed0e868e90..7df0e3e9ff 100644 --- a/tests/test_daemon_frontend.c +++ b/tests/test_daemon_frontend.c @@ -4,6 +4,7 @@ #include "test_framework.h" #include "test_helpers.h" +#include "daemon/application.h" #include "daemon/frontend.h" #include "daemon/ipc.h" #include "daemon/service.h" @@ -239,8 +240,18 @@ enum { * regression black-box: it must remain valid if the exact capacity changes * while still proving that overload cannot hide an already-pending EOF. */ FRONTEND_EOF_TEST_OVERFLOW_MESSAGES = 32, + /* Hold replacement context past the production frontend's 15 second + * no-progress EOF + drain so shutdown deterministically overlaps recovery. */ + FRONTEND_RECOVERY_EOF_RELEASE_MS = 16000, }; +typedef enum { + FRONTEND_RECOVERY_RETRY = 0, + FRONTEND_RECOVERY_CANCEL = 1, + FRONTEND_RECOVERY_CLEAN_EOF = 2, +} frontend_recovery_mode_t; + typedef struct { atomic_int requests; atomic_int session_cancels; @@ -268,6 +279,7 @@ typedef struct { typedef struct { int fd; bool overflow; + frontend_recovery_mode_t recovery_mode; frontend_eof_application_context_t *application; atomic_bool finished; atomic_bool succeeded; @@ -275,7 +287,9 @@ typedef struct { typedef struct { char conflict_log[FRONTEND_TEST_PATH_CAP]; + char build_fingerprint[CBM_DAEMON_BUILD_FINGERPRINT_SIZE]; cbm_daemon_ipc_endpoint_t *endpoint; + cbm_daemon_build_identity_t identity; cbm_version_cohort_manager_t *manager; cbm_daemon_runtime_service_t *service; cbm_daemon_runtime_client_t *client; @@ -337,16 +351,22 @@ static cbm_daemon_runtime_application_status_t frontend_eof_application_request( } /* Released: fall through and answer normally like every later item. */ } - if (request_length == 0) { + if (request_length == 0 || request[0] == CBM_DAEMON_APPLICATION_REQUEST_SET_CONTEXT) { return CBM_DAEMON_RUNTIME_APPLICATION_OK; } - uint8_t *response = malloc(request_length); + const uint8_t *response_source = request; + uint32_t response_length = request_length; + if (request[0] == CBM_DAEMON_APPLICATION_REQUEST_MCP) { + response_source++; + response_length--; + } + uint8_t *response = malloc(response_length); if (!response) { return CBM_DAEMON_RUNTIME_APPLICATION_HANDLER_ERROR; } - memcpy(response, request, request_length); + memcpy(response, response_source, response_length); *response_out = response; - *response_length_out = request_length; + *response_length_out = response_length; if (context->request_observed_fd >= 0) { const char marker = 'Q'; (void)write(context->request_observed_fd, &marker, 1); @@ -454,6 +474,86 @@ static void *frontend_eof_writer(void *opaque) { return NULL; } +static const char frontend_recovery_request[] = + "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/list\",\"params\":{}}\n"; +static const char frontend_recovery_cancel[] = + "{\"jsonrpc\":\"2.0\",\"method\":\"notifications/cancelled\"," + "\"params\":{\"requestId\":1}}\n"; +static const char frontend_recovery_followup[] = + "{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"tools/list\",\"params\":{}}\n"; + +static void *frontend_recovery_writer(void *opaque) { + frontend_eof_writer_t *writer = opaque; + bool ok = frontend_eof_write_all(writer->fd, frontend_recovery_request, + sizeof(frontend_recovery_request) - 1U); + uint64_t deadline = cbm_now_ms() + FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS; + if (writer->recovery_mode != FRONTEND_RECOVERY_RETRY) { + while (ok && + !atomic_load_explicit(&writer->application->first_request_started, + memory_order_acquire) && + cbm_now_ms() < deadline) { + cbm_usleep(1000); + } + ok = ok && atomic_load_explicit(&writer->application->first_request_started, + memory_order_acquire); + } + if (writer->recovery_mode == FRONTEND_RECOVERY_CANCEL) { + ok = ok && frontend_eof_write_all(writer->fd, frontend_recovery_cancel, + sizeof(frontend_recovery_cancel) - 1U); + /* Keep context blocked until the reader routes the cancellation. */ + cbm_usleep(100000); + atomic_store_explicit(&writer->application->release_first_request, true, + memory_order_release); + ok = ok && frontend_eof_write_all(writer->fd, frontend_recovery_followup, + sizeof(frontend_recovery_followup) - 1U); + } else if (writer->recovery_mode == FRONTEND_RECOVERY_CLEAN_EOF) { + ok = close(writer->fd) == 0 && ok; + writer->fd = -1; + cbm_usleep(FRONTEND_RECOVERY_EOF_RELEASE_MS * 1000U); + atomic_store_explicit(&writer->application->release_first_request, true, + memory_order_release); + } + int expected_requests = writer->recovery_mode == FRONTEND_RECOVERY_CLEAN_EOF ? 1 : 2; + while (ok && + atomic_load_explicit(&writer->application->requests, memory_order_acquire) < + expected_requests && + cbm_now_ms() < deadline) { + cbm_usleep(1000); + } + ok = ok && atomic_load_explicit(&writer->application->requests, memory_order_acquire) == + expected_requests; + if (writer->fd >= 0) { + ok = close(writer->fd) == 0 && ok; + writer->fd = -1; + } + atomic_store_explicit(&writer->succeeded, ok, memory_order_release); + atomic_store_explicit(&writer->finished, true, memory_order_release); + return NULL; +} + +static cbm_daemon_runtime_service_t *frontend_eof_service_start(frontend_eof_fixture_t *fixture) { + cbm_daemon_runtime_application_callbacks_t callbacks = { + .context = &fixture->application, + .session_open = frontend_eof_application_session_open, + .request = frontend_eof_application_request, + .request_cancel = frontend_eof_application_request_cancel, + .session_cancel = frontend_eof_application_session_cancel, + .session_close = frontend_eof_application_session_close, + }; + cbm_daemon_runtime_service_config_t config = { + .endpoint = fixture->endpoint, + .identity = fixture->identity, + .conflict_log_path = fixture->conflict_log, + .conflict_log_cap_bytes = 64U * 1024U, + .max_clients = 4, + .lease_timeout_ms = 5000, + .request_timeout_ms = FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS, + .shutdown_timeout_ms = FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS, + .application = callbacks, + }; + return cbm_daemon_runtime_service_start(&config); +} + static bool frontend_eof_fixture_start(frontend_eof_fixture_t *fixture, const char *parent) { memset(fixture, 0, sizeof(*fixture)); atomic_init(&fixture->application.requests, 0); @@ -467,53 +567,42 @@ static bool frontend_eof_fixture_start(frontend_eof_fixture_t *fixture, const ch fixture->application.request_observed_fd = -1; fixture->application.session_cancel_fd = -1; char key[CBM_DAEMON_KEY_SIZE]; - char build[CBM_DAEMON_BUILD_FINGERPRINT_SIZE]; int log_written = snprintf(fixture->conflict_log, sizeof(fixture->conflict_log), "%s/conflicts.ndjson", parent); if (log_written <= 0 || log_written >= (int)sizeof(fixture->conflict_log) || !cbm_daemon_rendezvous_key(key) || - !cbm_daemon_runtime_process_build_fingerprint((uint64_t)getpid(), build)) { + !cbm_daemon_runtime_process_build_fingerprint((uint64_t)getpid(), + fixture->build_fingerprint)) { return false; } fixture->endpoint = cbm_daemon_ipc_endpoint_new(key, parent); fixture->manager = fixture->endpoint ? cbm_version_cohort_manager_new(fixture->endpoint) : NULL; - cbm_daemon_build_identity_t identity = { + fixture->identity = (cbm_daemon_build_identity_t){ .semantic_version = "2.4.0", - .build_fingerprint = build, + .build_fingerprint = fixture->build_fingerprint, .cache_fingerprint = FRONTEND_TEST_CACHE, .protocol_abi = 3, .store_abi = 11, .feature_abi = 7, }; - cbm_daemon_runtime_application_callbacks_t callbacks = { - .context = &fixture->application, - .session_open = frontend_eof_application_session_open, - .request = frontend_eof_application_request, - .request_cancel = frontend_eof_application_request_cancel, - .session_cancel = frontend_eof_application_session_cancel, - .session_close = frontend_eof_application_session_close, - }; - cbm_daemon_runtime_service_config_t config = { - .endpoint = fixture->endpoint, - .identity = identity, - .conflict_log_path = fixture->conflict_log, - .conflict_log_cap_bytes = 64U * 1024U, - .max_clients = 4, - .lease_timeout_ms = 5000, - .request_timeout_ms = FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS, - .shutdown_timeout_ms = FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS, - .application = callbacks, - }; - fixture->service = fixture->manager ? cbm_daemon_runtime_service_start(&config) : NULL; + fixture->service = fixture->manager ? frontend_eof_service_start(fixture) : NULL; cbm_daemon_runtime_connect_result_t connect_result = {0}; fixture->client = fixture->service - ? cbm_daemon_runtime_client_connect(fixture->endpoint, &identity, + ? cbm_daemon_runtime_client_connect(fixture->endpoint, &fixture->identity, FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS, &connect_result) : NULL; return fixture->client && connect_result.status == CBM_DAEMON_RUNTIME_CONNECT_ACCEPTED; } +static bool frontend_eof_fixture_restart_service(frontend_eof_fixture_t *fixture) { + bool stopped = fixture->service && cbm_daemon_runtime_service_stop( + fixture->service, FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS); + bool freed = stopped && cbm_daemon_runtime_service_free(fixture->service); + fixture->service = freed ? frontend_eof_service_start(fixture) : NULL; + return stopped && freed && fixture->service; +} + static bool frontend_eof_fixture_finish(frontend_eof_fixture_t *fixture) { bool ok = true; if (fixture->client) { @@ -579,7 +668,21 @@ static int frontend_eof_child_run(const char *parent, bool overflow) { cbm_daemon_runtime_client_t *frontend_client = fixture.client; fixture.client = NULL; /* cbm_daemon_frontend_mcp_run consumes it. */ - int result = cbm_daemon_frontend_mcp_run(frontend_client, fixture.manager, input, output); + cbm_daemon_frontend_session_config_t session = { + .bootstrap = + { + .role = CBM_DAEMON_PROCESS_MCP_CLIENT, + .endpoint = fixture.endpoint, + .identity = &fixture.identity, + .executable_path = "unused", + .connect_timeout_ms = FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS, + .startup_timeout_ms = FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS, + }, + .session_root = parent, + .tool_profile = CBM_MCP_TOOL_PROFILE_ALL, + }; + int result = + cbm_daemon_frontend_mcp_run(frontend_client, fixture.manager, &session, input, output); bool joined = cbm_thread_join(&writer_thread) == 0; bool writer_ok = atomic_load_explicit(&writer.finished, memory_order_acquire) && atomic_load_explicit(&writer.succeeded, memory_order_acquire); @@ -660,6 +763,111 @@ static bool frontend_eof_run_isolated(const char *tag, bool overflow) { return child_ok && cleaned; } +static int frontend_recovery_child_run(const char *parent, frontend_recovery_mode_t recovery_mode) { + (void)alarm(FRONTEND_EOF_TEST_CATASTROPHIC_TIMEOUT_S); + frontend_eof_fixture_t fixture; + if (!frontend_eof_fixture_start(&fixture, parent)) { + (void)frontend_eof_fixture_finish(&fixture); + return 73; + } + atomic_store_explicit(&fixture.application.block_first_request, + recovery_mode != FRONTEND_RECOVERY_RETRY, memory_order_release); + cbm_daemon_runtime_client_t *stale_client = fixture.client; + fixture.client = NULL; + bool replacement_ready = frontend_eof_fixture_restart_service(&fixture); + int input_pipe[2] = {-1, -1}; + bool pipe_ready = replacement_ready && pipe(input_pipe) == 0; + FILE *input = pipe_ready ? fdopen(input_pipe[0], "rb") : NULL; + FILE *output = input ? tmpfile() : NULL; + frontend_eof_writer_t writer = { + .fd = input_pipe[1], + .recovery_mode = recovery_mode, + .application = &fixture.application, + }; + atomic_init(&writer.finished, false); + atomic_init(&writer.succeeded, false); + cbm_thread_t writer_thread; + bool writer_started = + output && cbm_thread_create(&writer_thread, 0, frontend_recovery_writer, &writer) == 0; + if (!writer_started && input_pipe[1] >= 0) { + (void)close(input_pipe[1]); + input_pipe[1] = -1; + } + bool request_ready = writer_started; + cbm_daemon_frontend_session_config_t session = { + .bootstrap = + { + .role = CBM_DAEMON_PROCESS_MCP_CLIENT, + .endpoint = fixture.endpoint, + .identity = &fixture.identity, + .executable_path = "unused", + .connect_timeout_ms = FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS, + .startup_timeout_ms = FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS, + }, + .session_root = parent, + .tool_profile = CBM_MCP_TOOL_PROFILE_ALL, + }; + int result = request_ready ? cbm_daemon_frontend_mcp_run(stale_client, fixture.manager, + &session, input, output) + : -1; + stale_client = NULL; /* the frontend consumes the stale or replacement handle */ + bool writer_joined = writer_started && cbm_thread_join(&writer_thread) == 0; + bool writer_ok = writer_joined && + atomic_load_explicit(&writer.finished, memory_order_acquire) && + atomic_load_explicit(&writer.succeeded, memory_order_acquire); + char response[512]; + memset(response, 0, sizeof(response)); + bool response_ready = result == 0 && fflush(output) == 0 && fseek(output, 0, SEEK_SET) == 0; + size_t response_length = response_ready ? fread(response, 1, sizeof(response) - 1U, output) : 0; + bool response_exact = false; + if (recovery_mode == FRONTEND_RECOVERY_CANCEL) { + response_exact = strstr(response, "\"id\":1") && strstr(response, "\"code\":-32800") && + strstr(response, frontend_recovery_followup); + } else if (recovery_mode == FRONTEND_RECOVERY_CLEAN_EOF) { + response_exact = response_length == 0; + } else { + response_exact = response_length == sizeof(frontend_recovery_request) - 1U && + memcmp(response, frontend_recovery_request, + sizeof(frontend_recovery_request) - 1U) == 0; + } + int requests = atomic_load_explicit(&fixture.application.requests, memory_order_acquire); + int expected_requests = recovery_mode == FRONTEND_RECOVERY_CLEAN_EOF ? 1 : 2; + bool input_closed = input && fclose(input) == 0; + bool output_closed = output && fclose(output) == 0; + bool fixture_closed = frontend_eof_fixture_finish(&fixture); + return replacement_ready && request_ready && result == 0 && response_exact && + requests == expected_requests && writer_ok && input_closed && output_closed && + fixture_closed + ? 0 + : 74; +} + +static bool frontend_recovery_run_isolated(frontend_recovery_mode_t recovery_mode) { + char parent[FRONTEND_TEST_PATH_CAP]; + int written = snprintf(parent, sizeof(parent), "%s/cbm-frontend-recovery-XXXXXX", cbm_tmpdir()); + if (written <= 0 || written >= (int)sizeof(parent) || !cbm_mkdtemp(parent)) { + return false; + } + pid_t child = fork(); + if (child == 0) { + (void)signal(SIGALRM, SIG_DFL); + _exit(frontend_recovery_child_run(parent, recovery_mode)); + } + int status = 0; + pid_t waited; + do { + waited = child > 0 ? waitpid(child, &status, 0) : -1; + } while (waited < 0 && errno == EINTR); + bool cleaned = th_rmtree(parent) == 0; + bool child_ok = child > 0 && waited == child && WIFEXITED(status) && WEXITSTATUS(status) == 0; + if (!child_ok && child > 0 && waited == child) { + (void)fprintf(stderr, "frontend recovery fixture failed: status=0x%x exit=%d signal=%d\n", + status, WIFEXITED(status) ? WEXITSTATUS(status) : -1, + WIFSIGNALED(status) ? WTERMSIG(status) : 0); + } + return child_ok && cleaned; +} + static void frontend_test_release_lease(cbm_version_cohort_lease_t **lease) { while (lease && *lease && cbm_version_cohort_lease_release(lease) != CBM_PRIVATE_FILE_LOCK_OK) { cbm_usleep(1000); @@ -920,7 +1128,20 @@ static int frontend_backpressure_frontend_run(const char *parent, int input_fd, return 81; } - int result = cbm_daemon_frontend_mcp_run(client, manager, input, output); + cbm_daemon_frontend_session_config_t session = { + .bootstrap = + { + .role = CBM_DAEMON_PROCESS_MCP_CLIENT, + .endpoint = endpoint, + .identity = &identity, + .executable_path = "unused", + .connect_timeout_ms = FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS, + .startup_timeout_ms = FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS, + }, + .session_root = parent, + .tool_profile = CBM_MCP_TOOL_PROFILE_ALL, + }; + int result = cbm_daemon_frontend_mcp_run(client, manager, &session, input, output); bool joined = cbm_thread_join(&writer_thread) == 0; bool writer_ok = atomic_load_explicit(&writer.finished, memory_order_acquire) && atomic_load_explicit(&writer.succeeded, memory_order_acquire); @@ -1397,8 +1618,21 @@ TEST(daemon_frontend_maintenance_exits_while_stdio_reader_is_blocked) { if (!announced) { _exit(70); } + cbm_daemon_frontend_session_config_t session = { + .bootstrap = + { + .role = CBM_DAEMON_PROCESS_MCP_CLIENT, + .endpoint = endpoint, + .identity = &identity, + .executable_path = "unused", + .connect_timeout_ms = FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS, + .startup_timeout_ms = FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS, + }, + .session_root = fixture.parent, + .tool_profile = CBM_MCP_TOOL_PROFILE_ALL, + }; int result = cbm_daemon_frontend_mcp_run((cbm_daemon_runtime_client_t *)(uintptr_t)1, - manager, input, output); + manager, &session, input, output); (void)result; _exit(71); } @@ -1612,6 +1846,27 @@ TEST(daemon_frontend_over_capacity_input_backpressures_without_loss) { PASS(); } +/* A request rejected before its application frame reaches the daemon is safe + * to replay. The frontend must replace its stale authenticated client, restore + * the session, reserve a token from the new connection, and return the one + * response without closing Codex's stdio transport. */ +TEST(daemon_frontend_recovers_one_unsent_request_after_daemon_replacement) { + ASSERT_TRUE(frontend_recovery_run_isolated(FRONTEND_RECOVERY_RETRY)); + PASS(); +} + +/* Context-replay cancellation must not strand a token or break the next request. */ +TEST(daemon_frontend_recovery_cancellation_preserves_next_request) { + ASSERT_TRUE(frontend_recovery_run_isolated(FRONTEND_RECOVERY_CANCEL)); + PASS(); +} + +/* Clean EOF during context replay must preserve worker-owned client cleanup. */ +TEST(daemon_frontend_clean_eof_during_recovery_preserves_client_ownership) { + ASSERT_TRUE(frontend_recovery_run_isolated(FRONTEND_RECOVERY_CLEAN_EOF)); + PASS(); +} + /* Clean EOF is the normal MCP-session ownership boundary. Accepted work gets a * bounded drain opportunity; work still active at the deadline is cancelled * with that exact session, while the frontend itself reports a clean close. */ @@ -1669,6 +1924,9 @@ SUITE(daemon_frontend) { RUN_TEST(daemon_local_participant_monitor_cancels_then_bounds_active_operation); RUN_TEST(daemon_local_participant_monitor_allows_supervisor_containment_window); RUN_TEST(daemon_frontend_over_capacity_input_backpressures_without_loss); + RUN_TEST(daemon_frontend_recovers_one_unsent_request_after_daemon_replacement); + RUN_TEST(daemon_frontend_recovery_cancellation_preserves_next_request); + RUN_TEST(daemon_frontend_clean_eof_during_recovery_preserves_client_ownership); RUN_TEST(daemon_frontend_eof_drain_timeout_cancels_and_returns_success); RUN_TEST(daemon_frontend_stdout_backpressure_eof_fail_stops_and_cancels_session); RUN_TEST(daemon_frontend_stdout_backpressure_maintenance_stops_and_cancels_session); diff --git a/tests/test_daemon_runtime.c b/tests/test_daemon_runtime.c index 991506a74b..b79d6c463b 100644 --- a/tests/test_daemon_runtime.c +++ b/tests/test_daemon_runtime.c @@ -907,7 +907,8 @@ static void *runtime_application_client_request_thread(void *opaque) { call->status = call->tagged ? cbm_daemon_runtime_client_application_request_tagged( call->client, call->request_token, call->request, call->request_length, - &call->response, &call->response_length, RUNTIME_TEST_TIMEOUT_MS) + &call->response, &call->response_length, NULL, + RUNTIME_TEST_TIMEOUT_MS) : cbm_daemon_runtime_client_application_request( call->client, call->request, call->request_length, &call->response, &call->response_length, RUNTIME_TEST_TIMEOUT_MS); @@ -3161,7 +3162,7 @@ TEST(daemon_runtime_presend_request_cancel_is_sticky_and_nonterminal) { cancel = cbm_daemon_runtime_client_application_cancel(client, request_token); cancelled_status = cbm_daemon_runtime_client_application_request_tagged( client, request_token, blocking_request, (uint32_t)sizeof(blocking_request), - &cancelled_response, &cancelled_response_length, RUNTIME_TEST_TIMEOUT_MS); + &cancelled_response, &cancelled_response_length, NULL, RUNTIME_TEST_TIMEOUT_MS); } if (cancelled_status == CBM_DAEMON_RUNTIME_APPLICATION_CANCELLED) { next_status = cbm_daemon_runtime_client_application_request( @@ -3218,8 +3219,10 @@ TEST(daemon_runtime_allows_only_one_unstarted_application_token) { bool first_reserved = false; bool duplicate_reservation_rejected = false; bool first_exact = false; + bool first_sent = false; bool second_reserved = false; bool second_exact = false; + bool second_sent = false; bool closed = false; bool exited = false; @@ -3239,7 +3242,7 @@ TEST(daemon_runtime_allows_only_one_unstarted_application_token) { cbm_daemon_runtime_application_status_t status = cbm_daemon_runtime_client_application_request_tagged( client, first_token, first_request, (uint32_t)sizeof(first_request), &response, - &response_length, RUNTIME_TEST_TIMEOUT_MS); + &response_length, &first_sent, RUNTIME_TEST_TIMEOUT_MS); first_exact = status == CBM_DAEMON_RUNTIME_APPLICATION_OK && response_length == sizeof(first_request) && response && memcmp(response, first_request, sizeof(first_request)) == 0; @@ -3255,7 +3258,7 @@ TEST(daemon_runtime_allows_only_one_unstarted_application_token) { cbm_daemon_runtime_application_status_t status = cbm_daemon_runtime_client_application_request_tagged( client, second_token, second_request, (uint32_t)sizeof(second_request), &response, - &response_length, RUNTIME_TEST_TIMEOUT_MS); + &response_length, &second_sent, RUNTIME_TEST_TIMEOUT_MS); second_exact = status == CBM_DAEMON_RUNTIME_APPLICATION_OK && response_length == sizeof(second_request) && response && memcmp(response, second_request, sizeof(second_request)) == 0; @@ -3274,9 +3277,11 @@ TEST(daemon_runtime_allows_only_one_unstarted_application_token) { ASSERT_TRUE(first_reserved); ASSERT_TRUE(duplicate_reservation_rejected); ASSERT_TRUE(first_exact); + ASSERT_TRUE(first_sent); ASSERT_TRUE(second_reserved); ASSERT_EQ(second_token, first_token + 1U); ASSERT_TRUE(second_exact); + ASSERT_TRUE(second_sent); ASSERT_TRUE(closed); ASSERT_TRUE(exited); ASSERT_EQ(atomic_load(&context.requests), 2); @@ -3376,6 +3381,10 @@ TEST(daemon_runtime_close_begin_retains_storage_and_rejects_late_exchange) { uint8_t *response = NULL; uint32_t response_length = 0; cbm_daemon_runtime_application_status_t status = CBM_DAEMON_RUNTIME_APPLICATION_OK; + cbm_daemon_runtime_application_token_t request_token = + CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID; + bool token_reserved = false; + bool request_sent = true; bool close_begun = false; bool duplicate_begin_rejected = false; bool close_acknowledged = false; @@ -3386,6 +3395,8 @@ TEST(daemon_runtime_close_begin_retains_storage_and_rejects_late_exchange) { RUNTIME_TEST_TIMEOUT_MS, &result); } if (client) { + token_reserved = + cbm_daemon_runtime_client_application_token_reserve(client, &request_token); close_begun = cbm_daemon_runtime_client_close_begin(client); duplicate_begin_rejected = close_begun && !cbm_daemon_runtime_client_close_begin(client); } @@ -3393,9 +3404,9 @@ TEST(daemon_runtime_close_begin_retains_storage_and_rejects_late_exchange) { /* Deterministically models the frontend boundary where close begins * after a worker claims an item but before it enters the runtime API. * The retained allocation must reject the call without touching IPC. */ - status = cbm_daemon_runtime_client_application_request( - client, request, (uint32_t)sizeof(request), &response, &response_length, - RUNTIME_TEST_TIMEOUT_MS); + status = cbm_daemon_runtime_client_application_request_tagged( + client, request_token, request, (uint32_t)sizeof(request), &response, &response_length, + &request_sent, RUNTIME_TEST_TIMEOUT_MS); close_acknowledged = cbm_daemon_runtime_client_close_finish(client, RUNTIME_TEST_TIMEOUT_MS); client = NULL; @@ -3413,11 +3424,13 @@ TEST(daemon_runtime_close_begin_retains_storage_and_rejects_late_exchange) { runtime_test_fixture_finish(&fixture); ASSERT_TRUE(started); + ASSERT_TRUE(token_reserved); ASSERT_TRUE(close_begun); ASSERT_TRUE(duplicate_begin_rejected); ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_TRANSPORT_ERROR); ASSERT_NULL(response); ASSERT_EQ(response_length, 0); + ASSERT_FALSE(request_sent); ASSERT_TRUE(close_acknowledged); ASSERT_TRUE(exited); ASSERT_EQ(atomic_load(&context.opened), 1); From b9da6dca03126ef0c8cacccb3321dd5d69ccf34d Mon Sep 17 00:00:00 2001 From: Pxx500 Date: Fri, 21 Aug 2026 18:44:15 +0200 Subject: [PATCH 02/27] test(frontend): widen recovery hang guard for MSan Signed-off-by: Pxx500 --- src/main.c | 1 - tests/test_daemon_frontend.c | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main.c b/src/main.c index 98db00561a..5547d79b15 100644 --- a/src/main.c +++ b/src/main.c @@ -3070,7 +3070,6 @@ int main(int argc, char **argv) { cbm_daemon_bootstrap_result_t bootstrap_result; cbm_daemon_bootstrap_status_t bootstrap_status = main_client_bootstrap_with_upgrade(&bootstrap_config, &bootstrap_result, true); - cbm_daemon_ipc_endpoint_free(endpoint); if (bootstrap_status != CBM_DAEMON_BOOTSTRAP_CONNECTED || !bootstrap_result.client) { main_report_client_bootstrap_failure(role, &bootstrap_result); cbm_daemon_ipc_endpoint_free(endpoint); diff --git a/tests/test_daemon_frontend.c b/tests/test_daemon_frontend.c index 7df0e3e9ff..96889ad330 100644 --- a/tests/test_daemon_frontend.c +++ b/tests/test_daemon_frontend.c @@ -231,6 +231,9 @@ enum { * dev machine (see the index-supervisor calibration). */ FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS = 30000, FRONTEND_EOF_TEST_CATASTROPHIC_TIMEOUT_S = 90, + /* Recovery can spend most of the generic guard restarting an instrumented + * daemon before the clean-EOF case enters its deliberate 15s drain. */ + FRONTEND_RECOVERY_TEST_CATASTROPHIC_TIMEOUT_S = 120, FRONTEND_BACKPRESSURE_MESSAGE_BYTES = 2 * 1024 * 1024, FRONTEND_BACKPRESSURE_FRONTEND_TIMEOUT_S = 90, FRONTEND_BACKPRESSURE_DAEMON_TIMEOUT_S = 90, @@ -764,7 +767,7 @@ static bool frontend_eof_run_isolated(const char *tag, bool overflow) { } static int frontend_recovery_child_run(const char *parent, frontend_recovery_mode_t recovery_mode) { - (void)alarm(FRONTEND_EOF_TEST_CATASTROPHIC_TIMEOUT_S); + (void)alarm(FRONTEND_RECOVERY_TEST_CATASTROPHIC_TIMEOUT_S); frontend_eof_fixture_t fixture; if (!frontend_eof_fixture_start(&fixture, parent)) { (void)frontend_eof_fixture_finish(&fixture); From 85482e3c599d454d76ccef5e862a5dc6c38a898e Mon Sep 17 00:00:00 2001 From: Pxx500 Date: Mon, 7 Sep 2026 17:30:13 +0200 Subject: [PATCH 03/27] test(frontend): address recovery review feedback Signed-off-by: Pxx500 --- src/daemon/frontend.c | 16 ++++++++ src/daemon/frontend.h | 12 ++++-- tests/test_daemon_frontend.c | 72 +++++++++++++++++++++++++----------- 3 files changed, 75 insertions(+), 25 deletions(-) diff --git a/src/daemon/frontend.c b/src/daemon/frontend.c index 92b0326867..d65284bd7a 100644 --- a/src/daemon/frontend.c +++ b/src/daemon/frontend.c @@ -119,6 +119,7 @@ static atomic_bool frontend_test_monitor_is_waiting = ATOMIC_VAR_INIT(false); static atomic_uint_fast64_t frontend_test_monitor_observation_count = ATOMIC_VAR_INIT(0); static atomic_uint_fast64_t frontend_test_worker_observation_count = ATOMIC_VAR_INIT(0); static atomic_uint_fast64_t frontend_test_worker_idle_count = ATOMIC_VAR_INIT(0); +static atomic_uint_fast64_t frontend_test_routed_cancellation_count = ATOMIC_VAR_INIT(0); void cbm_daemon_frontend_test_observer_reset(bool hold_monitor) { atomic_store_explicit(&frontend_test_monitor_observation_count, 0, memory_order_release); @@ -147,6 +148,10 @@ uint64_t cbm_daemon_frontend_test_worker_observations(void) { uint64_t cbm_daemon_frontend_test_worker_idle_cycles(void) { return atomic_load_explicit(&frontend_test_worker_idle_count, memory_order_acquire); } + +uint64_t cbm_daemon_frontend_test_routed_cancellations(void) { + return atomic_load_explicit(&frontend_test_routed_cancellation_count, memory_order_acquire); +} #endif static cbm_version_cohort_maintenance_presence_t frontend_observe_maintenance( @@ -478,6 +483,12 @@ static frontend_cancellation_route_t frontend_route_cancellation(frontend_state_ } cbm_mutex_unlock(&state->mutex); cbm_jsonrpc_request_free(&request); +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS + if (route == FRONTEND_CANCELLATION_ACTIVE || route == FRONTEND_CANCELLATION_QUEUED) { + atomic_fetch_add_explicit(&frontend_test_routed_cancellation_count, 1, + memory_order_release); + } +#endif return route; } @@ -531,6 +542,11 @@ static bool frontend_recover_client(frontend_state_t *state, frontend_item_t *it } cbm_daemon_bootstrap_result_t bootstrap = {0}; + /* Bootstrap can report launch failures on stderr from this worker. + * Keep that diagnostic + * to explain failed recovery. It never writes MCP + * stdout, but blocked stderr can delay + * exit. Do not add more logging here. */ cbm_daemon_bootstrap_status_t bootstrap_status = cbm_daemon_bootstrap_execute(&state->session->bootstrap, &bootstrap); cbm_daemon_runtime_client_t *replacement = diff --git a/src/daemon/frontend.h b/src/daemon/frontend.h index 10255687fd..9ee193be89 100644 --- a/src/daemon/frontend.h +++ b/src/daemon/frontend.h @@ -14,9 +14,11 @@ typedef struct cbm_daemon_maintenance_monitor cbm_daemon_maintenance_monitor_t; -/* Everything the stdio frontend must replay when its authenticated daemon - * connection is replaced. Pointer fields are borrowed for the complete - * frontend call. */ +/* Connection settings replayed when the authenticated daemon is replaced. + * Background activation + * from initialize is not restored here. + * Pointer fields are borrowed for the complete frontend + * call. */ typedef struct { cbm_daemon_bootstrap_config_t bootstrap; const char *session_root; @@ -67,6 +69,10 @@ bool cbm_daemon_frontend_test_monitor_waiting(void); uint64_t cbm_daemon_frontend_test_monitor_observations(void); uint64_t cbm_daemon_frontend_test_worker_observations(void); uint64_t cbm_daemon_frontend_test_worker_idle_cycles(void); +/* Counts matching cancellations marked under the frontend mutex. + * Tests snapshot this monotonic + * count before sending a notification. */ +uint64_t cbm_daemon_frontend_test_routed_cancellations(void); #endif /* Takes ownership of client and borrows cohort_manager for the complete call. diff --git a/tests/test_daemon_frontend.c b/tests/test_daemon_frontend.c index 96889ad330..acdaf05127 100644 --- a/tests/test_daemon_frontend.c +++ b/tests/test_daemon_frontend.c @@ -39,6 +39,8 @@ typedef struct { typedef struct { frontend_maintenance_fixture_t maintenance; char conflict_log[FRONTEND_TEST_PATH_CAP]; + char build_fingerprint[CBM_DAEMON_BUILD_FINGERPRINT_SIZE]; + cbm_daemon_build_identity_t identity; cbm_daemon_runtime_service_t *service; cbm_daemon_runtime_client_t *client; } frontend_idle_fixture_t; @@ -46,6 +48,7 @@ typedef struct { typedef struct { cbm_daemon_runtime_client_t *client; cbm_version_cohort_manager_t *manager; + const cbm_daemon_frontend_session_config_t *session; FILE *input; FILE *output; int result; @@ -99,22 +102,22 @@ static cbm_daemon_runtime_application_status_t frontend_idle_request( return CBM_DAEMON_RUNTIME_APPLICATION_OK; } -static void frontend_idle_request_cancel( - void *context, cbm_daemon_runtime_application_session_t *session, - cbm_daemon_runtime_application_token_t request_token) { +static void frontend_idle_request_cancel(void *context, + cbm_daemon_runtime_application_session_t *session, + cbm_daemon_runtime_application_token_t request_token) { (void)context; (void)session; (void)request_token; } -static void frontend_idle_session_cancel( - void *context, cbm_daemon_runtime_application_session_t *session) { +static void frontend_idle_session_cancel(void *context, + cbm_daemon_runtime_application_session_t *session) { (void)context; (void)session; } -static void frontend_idle_session_close( - void *context, cbm_daemon_runtime_application_session_t *session) { +static void frontend_idle_session_close(void *context, + cbm_daemon_runtime_application_session_t *session) { (void)context; (void)session; } @@ -128,22 +131,21 @@ static uint64_t frontend_test_process_id(void) { } static bool frontend_idle_fixture_start(frontend_idle_fixture_t *fixture) { - static const char cache[] = - "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"; + static const char cache[] = "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"; memset(fixture, 0, sizeof(*fixture)); if (!frontend_maintenance_fixture_start(&fixture->maintenance, "idle-observer")) { return false; } - char build[CBM_DAEMON_BUILD_FINGERPRINT_SIZE]; int log_written = snprintf(fixture->conflict_log, sizeof(fixture->conflict_log), "%s/conflicts.ndjson", fixture->maintenance.parent); if (log_written <= 0 || log_written >= (int)sizeof(fixture->conflict_log) || - !cbm_daemon_runtime_process_build_fingerprint(frontend_test_process_id(), build)) { + !cbm_daemon_runtime_process_build_fingerprint(frontend_test_process_id(), + fixture->build_fingerprint)) { return false; } - cbm_daemon_build_identity_t identity = { + fixture->identity = (cbm_daemon_build_identity_t){ .semantic_version = "2.4.0", - .build_fingerprint = build, + .build_fingerprint = fixture->build_fingerprint, .cache_fingerprint = cache, .protocol_abi = 3, .store_abi = 11, @@ -159,7 +161,7 @@ static bool frontend_idle_fixture_start(frontend_idle_fixture_t *fixture) { }; cbm_daemon_runtime_service_config_t config = { .endpoint = fixture->maintenance.endpoint, - .identity = identity, + .identity = fixture->identity, .conflict_log_path = fixture->conflict_log, .conflict_log_cap_bytes = 64U * 1024U, .max_clients = 2, @@ -170,10 +172,11 @@ static bool frontend_idle_fixture_start(frontend_idle_fixture_t *fixture) { }; fixture->service = cbm_daemon_runtime_service_start(&config); cbm_daemon_runtime_connect_result_t connect_result = {0}; - fixture->client = fixture->service - ? cbm_daemon_runtime_client_connect(fixture->maintenance.endpoint, &identity, - 30000, &connect_result) - : NULL; + fixture->client = + fixture->service + ? cbm_daemon_runtime_client_connect(fixture->maintenance.endpoint, &fixture->identity, + 30000, &connect_result) + : NULL; return fixture->client && connect_result.status == CBM_DAEMON_RUNTIME_CONNECT_ACCEPTED; } @@ -213,8 +216,8 @@ static int frontend_test_close_fd(int fd) { static void *frontend_idle_run(void *opaque) { frontend_idle_run_t *run = opaque; - run->result = - cbm_daemon_frontend_mcp_run(run->client, run->manager, run->input, run->output); + run->result = cbm_daemon_frontend_mcp_run(run->client, run->manager, run->session, run->input, + run->output); return NULL; } #endif @@ -479,11 +482,13 @@ static void *frontend_eof_writer(void *opaque) { static const char frontend_recovery_request[] = "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/list\",\"params\":{}}\n"; +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS static const char frontend_recovery_cancel[] = "{\"jsonrpc\":\"2.0\",\"method\":\"notifications/cancelled\"," "\"params\":{\"requestId\":1}}\n"; static const char frontend_recovery_followup[] = "{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"tools/list\",\"params\":{}}\n"; +#endif static void *frontend_recovery_writer(void *opaque) { frontend_eof_writer_t *writer = opaque; @@ -500,16 +505,24 @@ static void *frontend_recovery_writer(void *opaque) { ok = ok && atomic_load_explicit(&writer->application->first_request_started, memory_order_acquire); } +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS if (writer->recovery_mode == FRONTEND_RECOVERY_CANCEL) { + uint64_t cancellations = cbm_daemon_frontend_test_routed_cancellations(); ok = ok && frontend_eof_write_all(writer->fd, frontend_recovery_cancel, sizeof(frontend_recovery_cancel) - 1U); /* Keep context blocked until the reader routes the cancellation. */ - cbm_usleep(100000); + while (ok && cbm_daemon_frontend_test_routed_cancellations() == cancellations && + cbm_now_ms() < deadline) { + cbm_usleep(1000); + } + ok = ok && cbm_daemon_frontend_test_routed_cancellations() > cancellations; atomic_store_explicit(&writer->application->release_first_request, true, memory_order_release); ok = ok && frontend_eof_write_all(writer->fd, frontend_recovery_followup, sizeof(frontend_recovery_followup) - 1U); - } else if (writer->recovery_mode == FRONTEND_RECOVERY_CLEAN_EOF) { + } +#endif + if (writer->recovery_mode == FRONTEND_RECOVERY_CLEAN_EOF) { ok = close(writer->fd) == 0 && ok; writer->fd = -1; cbm_usleep(FRONTEND_RECOVERY_EOF_RELEASE_MS * 1000U); @@ -1537,9 +1550,20 @@ TEST(daemon_frontend_idle_uses_one_maintenance_observer) { bool pipe_ready = fixture_ready && cbm_pipe(input_pipe) == 0; FILE *input = pipe_ready ? frontend_test_fdopen_read(input_pipe[0]) : NULL; FILE *output = input ? tmpfile() : NULL; + cbm_daemon_frontend_session_config_t session = { + .bootstrap = {.role = CBM_DAEMON_PROCESS_MCP_CLIENT, + .endpoint = fixture.maintenance.endpoint, + .identity = &fixture.identity, + .executable_path = "unused", + .connect_timeout_ms = 30000, + .startup_timeout_ms = 30000}, + .session_root = fixture.maintenance.parent, + .tool_profile = CBM_MCP_TOOL_PROFILE_ALL, + }; frontend_idle_run_t run = { .client = fixture.client, .manager = fixture.maintenance.manager, + .session = &session, .input = input, .output = output, .result = -1, @@ -1859,10 +1883,12 @@ TEST(daemon_frontend_recovers_one_unsent_request_after_daemon_replacement) { } /* Context-replay cancellation must not strand a token or break the next request. */ +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS TEST(daemon_frontend_recovery_cancellation_preserves_next_request) { ASSERT_TRUE(frontend_recovery_run_isolated(FRONTEND_RECOVERY_CANCEL)); PASS(); } +#endif /* Clean EOF during context replay must preserve worker-owned client cleanup. */ TEST(daemon_frontend_clean_eof_during_recovery_preserves_client_ownership) { @@ -1928,7 +1954,9 @@ SUITE(daemon_frontend) { RUN_TEST(daemon_local_participant_monitor_allows_supervisor_containment_window); RUN_TEST(daemon_frontend_over_capacity_input_backpressures_without_loss); RUN_TEST(daemon_frontend_recovers_one_unsent_request_after_daemon_replacement); +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS RUN_TEST(daemon_frontend_recovery_cancellation_preserves_next_request); +#endif RUN_TEST(daemon_frontend_clean_eof_during_recovery_preserves_client_ownership); RUN_TEST(daemon_frontend_eof_drain_timeout_cancels_and_returns_success); RUN_TEST(daemon_frontend_stdout_backpressure_eof_fail_stops_and_cancels_session); From 71ac6ecc15d3ab3ebcd31f0e467071a506006ca2 Mon Sep 17 00:00:00 2001 From: Pxx500 Date: Sat, 26 Sep 2026 23:01:11 +0200 Subject: [PATCH 04/27] fix(mcp): reuse response cleanup in frontend worker Signed-off-by: Pxx500 --- src/daemon/frontend.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/daemon/frontend.c b/src/daemon/frontend.c index d65284bd7a..a4d4f88467 100644 --- a/src/daemon/frontend.c +++ b/src/daemon/frontend.c @@ -580,6 +580,12 @@ static bool frontend_recover_client(frontend_state_t *state, frontend_item_t *it return usable; } +static void frontend_response_reset(uint8_t **response, uint32_t *response_length) { + free(*response); + *response = NULL; + *response_length = 0; +} + static void *frontend_worker(void *opaque) { frontend_state_t *state = opaque; for (;;) { @@ -606,9 +612,7 @@ static void *frontend_worker(void *opaque) { state->client, item.request_token, item.message, &response, &response_length, &request_sent, FRONTEND_REQUEST_TIMEOUT_MS); if (status == CBM_DAEMON_RUNTIME_APPLICATION_TRANSPORT_ERROR && !request_sent) { - free(response); - response = NULL; - response_length = 0; + frontend_response_reset(&response, &response_length); bool cancelled = false; if (frontend_recover_client(state, &item, &cancelled)) { if (cancelled) { @@ -632,7 +636,7 @@ static void *frontend_worker(void *opaque) { item.content_length_framed); } } - free(response); + frontend_response_reset(&response, &response_length); bool expected_stop = failed && frontend_should_stop(state); frontend_end_request(state, failed && !expected_stop); frontend_item_free(&item); From f200d7477f192caedaa5b4cda7d9eccc2022a069 Mon Sep 17 00:00:00 2001 From: Andrew Studnicky Date: Sun, 27 Sep 2026 18:38:20 -0400 Subject: [PATCH 05/27] fix(extraction): attribute Swift lifecycle calls Signed-off-by: Andrew Studnicky --- internal/cbm/extract_defs.c | 5 + internal/cbm/helpers.c | 10 ++ internal/cbm/lang_specs.c | 3 +- tests/test_extraction.c | 283 ++++++++++++++++++++++++++++++++++-- 4 files changed, 290 insertions(+), 11 deletions(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 916f1136f9..9db81506cb 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -210,6 +210,11 @@ static void extract_elixir_call(CBMExtractCtx *ctx, TSNode node, const CBMLangSp // Get "name" field from a node static TSNode func_name_node(TSNode node) { + const char *node_type = ts_node_type(node); + if (strcmp(node_type, "init_declaration") == 0 || + strcmp(node_type, "deinit_declaration") == 0) { + return node; + } TSNode name = ts_node_child_by_field_name(node, TS_FIELD("name")); if (ts_node_is_null(name)) { /* Protobuf rpc: name is in rpc_name child, not "name" field */ diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index 50c75bdf2e..1a51dc971a 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -959,6 +959,16 @@ TSNode cbm_resolve_c_declarator_name_node(TSNode func_node) { // const", and a member lookup for "operator bool" (the implicit call in // `if (obj)`) misses. char *cbm_func_name_node_text(CBMArena *a, TSNode name_node, const char *source, CBMLanguage lang) { + if (lang == CBM_LANG_SWIFT) { + const char *kind = ts_node_type(name_node); + if (strcmp(kind, "init_declaration") == 0) { + return cbm_arena_strdup(a, "init"); + } + if (strcmp(kind, "deinit_declaration") == 0) { + return cbm_arena_strdup(a, "deinit"); + } + } + char *text = cbm_node_text(a, name_node, source); if (text && strcmp(ts_node_type(name_node), "operator_cast") == 0) { char *paren = strchr(text, '('); diff --git a/internal/cbm/lang_specs.c b/internal/cbm/lang_specs.c index 3a5fad0d46..8b29c9005a 100644 --- a/internal/cbm/lang_specs.c +++ b/internal/cbm/lang_specs.c @@ -571,7 +571,8 @@ static const char *objc_assign_types[] = {"assignment_expression", NULL}; // well as in the name resolvers — extract_class_methods gates on this set, so // without the entry the requirement is walked and then discarded. static const char *swift_func_types[] = {"function_declaration", "protocol_function_declaration", - "macro_declaration", NULL}; + "macro_declaration", "init_declaration", + "deinit_declaration", NULL}; // KNOWN GAP: struct_declaration and enum_declaration are not node types in the // vendored tree-sitter-swift grammar — it models both as class_declaration — so // these two entries never match anything, and a bare Swift `enum` is labeled diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 1e6506ed1a..80cbd59e51 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -69,6 +69,30 @@ static int has_def_qn(CBMFileResult *r, const char *qn) { return 0; } +/* Return the definition with the given qualified name, or NULL. Distinct from + * find_def_by_name for the same reason has_def_qn is distinct from has_def. */ +/* Find a definition by its qualified name relative to the file's module QN + * (e.g. "Box.init" resolves to ".Box.init"). */ +static const CBMDefinition *find_def_in_module(CBMFileResult *r, const char *rel_qn) { + char qn[512]; + snprintf(qn, sizeof(qn), "%s.%s", r->module_qn, rel_qn); + for (int i = 0; i < r->defs.count; i++) { + if (r->defs.items[i].qualified_name && strcmp(r->defs.items[i].qualified_name, qn) == 0) + return &r->defs.items[i]; + } + return NULL; +} + +/* Find an in-body call by its raw callee text; returns the call or NULL. */ +static const CBMCall *find_call_by_callee(CBMFileResult *r, const char *callee) { + for (int i = 0; i < r->calls.count; i++) { + if (r->calls.items[i].callee_name && strcmp(r->calls.items[i].callee_name, callee) == 0) { + return &r->calls.items[i]; + } + } + return NULL; +} + static int count_defs_with_label(CBMFileResult *r, const char *label) { int count = 0; for (int i = 0; i < r->defs.count; i++) { @@ -2004,6 +2028,247 @@ TEST(swift_force_unwrap_scanner_shift) { PASS(); } +/* #2379: a struct's plain `init` is a callable, not a name-less hole -- a call + * inside it must source from the initializer, not fall back to the file's + * Module scope. */ +TEST(swift_init_struct_issue2379) { + CBMFileResult *r = extract("func helper() -> Int { 1 }\n" + "struct Box {\n" + " let value: Int\n" + " init() {\n" + " value = helper()\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Box.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *init_def = find_def_in_module(r, "Box.init"); + ASSERT_NOT_NULL(init_def); + ASSERT_STR_EQ(init_def->label, "Method"); + ASSERT_STR_EQ(init_def->name, "init"); + + const CBMCall *call = find_call_by_callee(r, "helper"); + ASSERT_NOT_NULL(call); + ASSERT_NOT_NULL(call->enclosing_func_qn); + ASSERT_STR_EQ(call->enclosing_func_qn, init_def->qualified_name); + ASSERT_STR_NEQ(call->enclosing_func_qn, r->module_qn); + + cbm_free_result(r); + PASS(); +} + +/* #2379: a class's designated init takes parameters -- coverage that a + * parameterized init is extracted the same way as a bare one. */ +TEST(swift_init_class_designated_issue2379) { + CBMFileResult *r = extract("func helper() -> Int { 1 }\n" + "final class Holder {\n" + " let value: Int\n" + " init(seed: Int) {\n" + " value = helper() + seed\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Holder.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *init_def = find_def_in_module(r, "Holder.init"); + ASSERT_NOT_NULL(init_def); + ASSERT_STR_EQ(init_def->label, "Method"); + + const CBMCall *call = find_call_by_callee(r, "helper"); + ASSERT_NOT_NULL(call); + ASSERT_NOT_NULL(call->enclosing_func_qn); + ASSERT_STR_EQ(call->enclosing_func_qn, init_def->qualified_name); + ASSERT_STR_NEQ(call->enclosing_func_qn, r->module_qn); + + cbm_free_result(r); + PASS(); +} + +/* #2379: a convenience init delegates via `self.init(...)`. Its call to + * `other()` (an argument of the delegating call) must source from the + * initializer, not from the file. */ +TEST(swift_init_convenience_delegates_issue2379) { + CBMFileResult *r = extract("func other() -> Int { 2 }\n" + "final class Holder {\n" + " let value: Int\n" + " init(seed: Int) {\n" + " value = seed\n" + " }\n" + " convenience init() {\n" + " self.init(seed: other())\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Holder.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *init_def = find_def_in_module(r, "Holder.init"); + ASSERT_NOT_NULL(init_def); + + /* self.init(...) is a plain member call today, same as any other + * `receiver.member(...)` invocation -- no dedicated delegation handling. */ + ASSERT(has_call(r, "self.init")); + + const CBMCall *call = find_call_by_callee(r, "other"); + ASSERT_NOT_NULL(call); + ASSERT_NOT_NULL(call->enclosing_func_qn); + ASSERT_STR_EQ(call->enclosing_func_qn, init_def->qualified_name); + ASSERT_STR_NEQ(call->enclosing_func_qn, r->module_qn); + + cbm_free_result(r); + PASS(); +} + +/* #2379: two initializers in the same type calling the SAME callee must both + * attribute to the type's init, not to the file's Module scope. Both share + * the QN `Holder.init` -- codebase-memory-mcp already collapses overloaded + * Swift methods onto one node (DeusData/codebase-memory-mcp#2061), and an + * init is no exception; distinguishing overloads by signature is that + * issue's concern, not this one's. */ +TEST(swift_init_multiple_not_collapsed_issue2379) { + CBMFileResult *r = extract("func shared() -> Int { 3 }\n" + "final class Holder {\n" + " let value: Int\n" + " init(seed: Int) {\n" + " value = shared()\n" + " }\n" + " convenience init() {\n" + " self.init(seed: shared())\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Holder.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *init_def = find_def_in_module(r, "Holder.init"); + ASSERT_NOT_NULL(init_def); + + int from_init = 0; + for (int i = 0; i < r->calls.count; i++) { + const CBMCall *c = &r->calls.items[i]; + if (!c->callee_name || strcmp(c->callee_name, "shared") != 0 || !c->enclosing_func_qn) { + continue; + } + ASSERT_STR_EQ(c->enclosing_func_qn, init_def->qualified_name); + ASSERT_STR_NEQ(c->enclosing_func_qn, r->module_qn); + from_init++; + } + ASSERT_EQ(from_init, 2); + + cbm_free_result(r); + PASS(); +} + +/* #2379: a failable `init?` in an enum is still a callable initializer. */ +TEST(swift_init_failable_enum_issue2379) { + CBMFileResult *r = extract("func helper() -> Int { 1 }\n" + "enum Mode {\n" + " case on\n" + " init?(flag: Bool) {\n" + " if helper() > 0 { self = .on } else { return nil }\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Mode.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *init_def = find_def_in_module(r, "Mode.init"); + ASSERT_NOT_NULL(init_def); + + const CBMCall *call = find_call_by_callee(r, "helper"); + ASSERT_NOT_NULL(call); + ASSERT_NOT_NULL(call->enclosing_func_qn); + ASSERT_STR_EQ(call->enclosing_func_qn, init_def->qualified_name); + ASSERT_STR_NEQ(call->enclosing_func_qn, r->module_qn); + + cbm_free_result(r); + PASS(); +} + +/* #2379: `extension X { }` is modeled by the vendored grammar as a + * class_declaration named X, so an init declared in an extension goes through + * the same class-method path as one declared in the primary type. */ +TEST(swift_init_extension_issue2379) { + CBMFileResult *r = extract("func other() -> Int { 2 }\n" + "extension Box {\n" + " init(raw: Int) {\n" + " value = other()\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Box+Raw.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *init_def = find_def_in_module(r, "Box.init"); + ASSERT_NOT_NULL(init_def); + + const CBMCall *call = find_call_by_callee(r, "other"); + ASSERT_NOT_NULL(call); + ASSERT_NOT_NULL(call->enclosing_func_qn); + ASSERT_STR_EQ(call->enclosing_func_qn, init_def->qualified_name); + ASSERT_STR_NEQ(call->enclosing_func_qn, r->module_qn); + + cbm_free_result(r); + PASS(); +} + +/* #2379: regression guard -- an ordinary method in the same type keeps its own + * attribution unchanged (this already worked pre-fix; the initializer fix + * must not disturb it). */ +TEST(swift_method_still_attributed_issue2379) { + CBMFileResult *r = extract("func other() -> Int { 2 }\n" + "struct Box {\n" + " init() {}\n" + " func read() -> Int {\n" + " other()\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Box.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *read_def = find_def_in_module(r, "Box.read"); + ASSERT_NOT_NULL(read_def); + + const CBMCall *call = find_call_by_callee(r, "other"); + ASSERT_NOT_NULL(call); + ASSERT_NOT_NULL(call->enclosing_func_qn); + ASSERT_STR_EQ(call->enclosing_func_qn, read_def->qualified_name); + + cbm_free_result(r); + PASS(); +} + +/* #2379: a `deinit` has the same gap as `init` -- no source-level name, so a + * call inside its body attributed to the file's Module scope instead of the + * type. */ +TEST(swift_deinit_issue2379) { + CBMFileResult *r = extract("func helper() -> Int { 1 }\n" + "final class Deinitable {\n" + " deinit {\n" + " _ = helper()\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Deinitable.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *deinit_def = find_def_in_module(r, "Deinitable.deinit"); + ASSERT_NOT_NULL(deinit_def); + ASSERT_STR_EQ(deinit_def->name, "deinit"); + + const CBMCall *call = find_call_by_callee(r, "helper"); + ASSERT_NOT_NULL(call); + ASSERT_NOT_NULL(call->enclosing_func_qn); + ASSERT_STR_EQ(call->enclosing_func_qn, deinit_def->qualified_name); + ASSERT_STR_NEQ(call->enclosing_func_qn, r->module_qn); + + cbm_free_result(r); + PASS(); +} + /* --- Objective-C --- */ TEST(objc_interface) { CBMFileResult *r = @@ -4528,16 +4793,6 @@ TEST(extract_ts_decorators_survive_interleaved_comment) { PASS(); } -/* Find an in-body call by its raw callee text; returns the call or NULL. */ -static const CBMCall *find_call_by_callee(CBMFileResult *r, const char *callee) { - for (int i = 0; i < r->calls.count; i++) { - if (r->calls.items[i].callee_name && strcmp(r->calls.items[i].callee_name, callee) == 0) { - return &r->calls.items[i]; - } - } - return NULL; -} - /* #1892: the Swift grammar declares no "arguments" field, so the generic field * lookup read nothing and every Swift call lost its arguments. Without the URL * the service-pattern table cannot raise an HTTP_CALLS edge or a Route node, @@ -8442,6 +8697,14 @@ SUITE(extraction) { RUN_TEST(swift_constructor_call); RUN_TEST(swift_chained_call); RUN_TEST(swift_force_unwrap_scanner_shift); + RUN_TEST(swift_init_struct_issue2379); + RUN_TEST(swift_init_class_designated_issue2379); + RUN_TEST(swift_init_convenience_delegates_issue2379); + RUN_TEST(swift_init_multiple_not_collapsed_issue2379); + RUN_TEST(swift_init_failable_enum_issue2379); + RUN_TEST(swift_init_extension_issue2379); + RUN_TEST(swift_method_still_attributed_issue2379); + RUN_TEST(swift_deinit_issue2379); RUN_TEST(swift_call_string_arg_issue1892); RUN_TEST(swift_nested_url_constructor_issue1892); RUN_TEST(swift_nested_url_no_bang_issue1892); From 7aeacae908e1846501a559223453b99aacfb37a7 Mon Sep 17 00:00:00 2001 From: Your GitHub Username Date: Sat, 3 Oct 2026 08:03:47 +0530 Subject: [PATCH 06/27] fix(ci): parse clang-tidy analyzer warning suffix Signed-off-by: Your GitHub Username --- scripts/lint-mem-gate.py | 10 +++++++++- tests/test_lint_mem_gate.sh | 39 +++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100755 tests/test_lint_mem_gate.sh diff --git a/scripts/lint-mem-gate.py b/scripts/lint-mem-gate.py index 140062f372..d5ccd76d2d 100755 --- a/scripts/lint-mem-gate.py +++ b/scripts/lint-mem-gate.py @@ -47,7 +47,15 @@ MIN_WHY = 120 MIN_TRIED = 40 -FINDING_RE = re.compile(r"^(?P[^:]+):(?P\d+):\d+:\s+(?:error|warning):\s+(?P.*?)\s*\[(?P[\w.-]+)\]\s*$") +# With WarningsAsErrors enabled, clang-tidy appends the escalation reason to +# analyzer checks, for example: +# [clang-analyzer-core.NullDereference,-warnings-as-errors] +# Keep the check name separate from that suffix so the gate still matches the +# same whitelist key instead of silently dropping the finding. +FINDING_RE = re.compile( + r"^(?P[^:]+):(?P\d+):\d+:\s+(?:error|warning):\s+" + r"(?P.*?)\s*\[(?P[\w.-]+)(?:,-warnings-as-errors)?\]\s*$" +) def strip_noise(text): diff --git a/tests/test_lint_mem_gate.sh b/tests/test_lint_mem_gate.sh new file mode 100755 index 0000000000..05c2bd4a3a --- /dev/null +++ b/tests/test_lint_mem_gate.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Regression contract for clang-tidy's `,-warnings-as-errors` diagnostic suffix. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +FIX="$(mktemp -d "${TMPDIR:-/tmp}/lint-mem-gate.XXXXXX")" +trap 'rm -rf "$FIX"' EXIT + +cat > "$FIX/sample.c" <<'EOF' +int sample(int *value) { + return *value; +} +EOF + +cat > "$FIX/whitelist.txt" < "$FIX/stderr" + +grep -q 'memory gate clean (1 argued false positive' "$FIX/stderr" +echo 'lint-mem-gate warning suffix contract passed' From de1615264aa7ec2ef997d8e73fede1f0abc71965 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Wed, 19 Aug 2026 14:39:38 +0800 Subject: [PATCH 07/27] feat(index): add worker resource watchdogs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discovery limits bound what indexing accepts, not what it then costs. A repository well inside those bounds can still exhaust the host through parser memory, or simply never finish, and a supervised worker that hangs leaves the parent waiting with nothing to report. Add index_max_rss_mb and index_max_duration_seconds, enforced by the parent against the worker process tree rather than the worker process alone, so a runaway child cannot hide behind a small parent. Resident memory is sampled through the platform interface on macOS, Linux and Windows. Crossing a limit terminates the tree and yields one trusted, structured terminal result that attributes the failure to the resource that caused it. Both limits default to off. A measurement that cannot be taken fails the attempt instead of passing it: a watchdog that quietly stops watching is worse than no watchdog at all. The shell fixture that stands in for the supervisor names the two new keys. The worker accepts only a policy that spells out every key it knows, which is what keeps a stale supervisor from starting a worker it cannot bound. Signed-off-by: 刘冲 --- README.md | 4 +- docs/CONFIGURATION.md | 10 +- docs/INDEX_RESOURCE_LIMITS.md | 48 +++++- scripts/test-runtime.sh | 3 +- src/cli/cli.c | 10 +- src/daemon/application.c | 20 ++- src/foundation/index_policy.c | 126 +++++++++----- src/foundation/index_policy.h | 11 ++ src/foundation/subprocess.c | 285 ++++++++++++++++++++++++++++++++ src/foundation/subprocess.h | 18 ++ src/mcp/index_supervisor.c | 267 ++++++++++++++++++++++++++++-- src/mcp/index_supervisor.h | 38 ++++- src/mcp/mcp.c | 114 ++++++++++--- src/mcp/mcp.h | 1 + src/mcp/mcp_internal.h | 4 + src/pipeline/pipeline.c | 5 +- tests/test_daemon_application.c | 133 +++++++++++++-- tests/test_index_policy.c | 72 +++++++- tests/test_index_supervisor.c | 269 ++++++++++++++++++++++++++++++ tests/test_mcp.c | 98 +++++++++++ tests/test_subprocess.c | 47 ++++++ 21 files changed, 1472 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index 590c9eef96..4466dae50b 100644 --- a/README.md +++ b/README.md @@ -928,10 +928,12 @@ codebase-memory-mcp config set watcher_enabled false # stop the watcher thre codebase-memory-mcp config set watch_non_git true # also poll non-git project roots (default: false) codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit +codebase-memory-mcp config set index_max_rss_mb 8192 # optional worker-tree current RSS limit +codebase-memory-mcp config set index_max_duration_seconds 3600 # optional total worker duration codebase-memory-mcp config reset auto_index # reset to default ``` -The two `index_max_*` settings default to `off`. Exceeding one fails the complete +The four `index_max_*` settings default to `off`. Exceeding one fails the complete index attempt rather than publishing a partial graph; an existing serving index is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 3e9956d601..3d3e9a9716 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -93,6 +93,8 @@ Current keys: | `watch_non_git` | `false` | Also poll project roots that are **not git repositories**. By default the watcher only follows git projects, so a project indexed from a plain directory is never refreshed after its first index — reindex it manually with `index_repository`. Set `true` to poll such roots on the same adaptive cadence with a file-tree scan: the indexer's own discovery walk (same skip lists, `.gitignore` and `.cbmignore` rules) hashed over each file's path, size and mtime. Any change reindexes once; paths the indexer skips (including cbm's own `.codebase-memory/` output) never trigger. The first poll after the daemon starts reindexes each such project once, since nothing records which tree state the index holds. The scan walks the whole tree every poll, so it costs more than git polling on very large trees. Read once when the daemon starts, like `watcher_enabled`. | | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | +| `index_max_rss_mb` | `off` | Optional maximum current RSS in MiB for the complete contained index-worker process tree (`64..1048576`). | +| `index_max_duration_seconds` | `off` | Optional maximum total worker duration in seconds (`1..86400`). | For a watched project, `index_status` names the polling method in `watch.strategy`: `git` (HEAD and dirty-state polling), `tree` (a non-git root polled by the @@ -126,11 +128,13 @@ has not run yet). > `auto_index` still runs, and `index_repository` stays available for manual > reindexing. -The two `index_max_*` settings are independent and disabled by default. They +The four `index_max_*` settings are independent and disabled by default. They apply to explicit indexing, automatic indexing, and watcher re-indexing, but not to `cross-repo-intelligence`, which does not scan repository source files. -Equality is allowed; exceeding either setting fails the complete index request -and preserves any previously serving database. See +Equality is allowed; exceeding any setting fails the complete index request and +preserves any previously serving database. Worker RSS covers descendants and is +not the same as the internal `CBM_MEM_BUDGET_MB` allocation budget. Total +duration is independent of the existing 15-minute no-log-progress timeout. See [Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and error-response details. diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index a3d57e0155..66be4ecf46 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -1,8 +1,8 @@ # Index resource limits Index resource limits are optional operator controls for repositories whose -discovery breadth is not known in advance. They are disabled by default so -existing large-repository workloads retain their current behavior. +discovery breadth or worker runtime is not known in advance. They are disabled +by default so existing large-repository workloads retain their current behavior. ## Discovery settings @@ -23,6 +23,39 @@ Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero, negative values, suffixes, trailing characters, and values outside the stated ranges are rejected without changing the stored value. +## Worker settings + +| Key | Default | Accepted value | Protects | +|---|---:|---:|---| +| `index_max_rss_mb` | `off` | `off` or `64..1048576` | Current RSS of the complete worker process tree | +| `index_max_duration_seconds` | `off` | `off` or `1..86400` | Total worker wall-clock duration | + +```bash +codebase-memory-mcp config set index_max_rss_mb 8192 +codebase-memory-mcp config set index_max_duration_seconds 3600 +``` + +RSS is the current resident memory of the contained worker and every descendant, +not the worker's allocation budget and not peak memory. This hard watchdog is +separate from the internal `CBM_MEM_BUDGET_MB` soft budget. The supervisor +samples RSS at most once every 250 milliseconds so the watchdog does not turn +full process-table enumeration into a busy loop. + +Duration uses a monotonic clock from successful spawn. It is independent of the +existing 15-minute quiet timeout: continuous log progress does not reset total +duration, while the quiet timeout continues to identify a worker that stops +making progress. + +Equality is allowed. The first RSS or elapsed-duration observation above its +limit starts the existing graceful-to-force process-tree shutdown. CBM reports +terminal only after the tree is quiescent or a bounded containment failure is +explicitly surfaced. Resource termination is not retried and does not +quarantine a source file. + +If RSS is enabled and three consecutive probes cannot obtain any trustworthy +tree measurement while the root worker is still running, CBM fails closed with +`code=resource_probe_failed`. + ## Counting and failure semantics `index_max_files` counts a file only after it passes directory pruning, ignore @@ -55,6 +88,12 @@ The previous database remains available because publication occurs only after a complete discovery and successful staged build. If no previous database exists, `serving_index_preserved` is false. +Worker limit failures use the same shape with `stage=worker`, +`resource=rss_bytes` and `unit=bytes`, or `resource=duration_ms` and +`unit=milliseconds`. RSS measurement failures use `code=resource_probe_failed` +and omit `observed`, `limit`, and `unit` because no trustworthy observation was +available. + ## Trust and compatibility Limits are read from the CLI-managed `_config.db`; they are not MCP request @@ -64,5 +103,6 @@ parent policy. These settings do not replace or increase `auto_index_limit`, change the 512 MiB single-file cap, alter workspace-root authorization, or affect -`cross-repo-intelligence`. With both settings `off`, discovery follows the -existing unbounded path. +`cross-repo-intelligence`. With all settings `off`, discovery follows the +existing path and the supervisor performs no periodic RSS probe or total-duration +termination. diff --git a/scripts/test-runtime.sh b/scripts/test-runtime.sh index 5d2c0c3aee..e6e60b15a7 100644 --- a/scripts/test-runtime.sh +++ b/scripts/test-runtime.sh @@ -112,7 +112,8 @@ _cbm_test_runtime_daemon() { # cbm_mcp_index_policy_add_to_args: every key in cbm_index_policy_key_at, and # nothing else. cbm_test_index_worker_policy_json() { - printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"}' + printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"' + printf '%s' ',"index_max_rss_mb":"off","index_max_duration_seconds":"off"}' } cbm_test_runtime_cleanup() { diff --git a/src/cli/cli.c b/src/cli/cli.c index 1ebe1687e8..00b2db3410 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -7603,6 +7603,8 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"}, {CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"}, {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"}, + {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree RSS MiB, or off"}, + {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "off", "Max worker duration in seconds, or off"}, }; /* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing @@ -7629,8 +7631,12 @@ static bool config_key_is_ui(const char *key) { } static bool config_key_is_index_policy(const char *key) { - return key && (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0 || - strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); + for (size_t index = 0; key && index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, cbm_index_policy_key_at(index)) == 0) { + return true; + } + } + return false; } static int config_index_policy_write(cbm_config_t *config, const char *key, const char *value) { diff --git a/src/daemon/application.c b/src/daemon/application.c index 22b6f0d1a7..a7b2f4f5b0 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -363,6 +363,14 @@ static int application_worker_start_default(void *context, const char *args_json const char *quarantine_file, cbm_daemon_application_worker_t *worker_out) { (void)context; + cbm_index_resource_policy_t resource_policy; + char error[CBM_SZ_256] = {0}; + if (!cbm_mcp_index_policy_from_internal_args(args_json, &resource_policy, error, + sizeof(error))) { + cbm_log_error("daemon.index.policy", "error", error); + *worker_out = NULL; + return -1; + } application_default_worker_t *worker = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*worker)); if (!worker) { return -1; @@ -372,9 +380,9 @@ static int application_worker_start_default(void *context, const char *args_json const char *root = yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(args_doc), "repo_path")); worker->project = application_index_project_key(root, args_json); yyjson_doc_free(args_doc); - int result = cbm_index_worker_start_with_log(args_json, memory_budget_bytes, false, marker_file, - quarantine_file, application_worker_diagnostic, - worker, &worker->handle); + int result = cbm_index_worker_start_with_policy_log( + args_json, memory_budget_bytes, &resource_policy, false, marker_file, quarantine_file, + application_worker_diagnostic, worker, &worker->handle); if (result != 0) { application_worker_destroy_default(NULL, worker); *worker_out = NULL; @@ -1567,6 +1575,12 @@ static application_attempt_decision_t application_consume_attempt( application_attempt_free(attempt); return APPLICATION_ATTEMPT_DECISION_SUCCESS; } + if (disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + execution->response = + cbm_mcp_index_worker_resource_response(job->args_json, &attempt->result); + application_attempt_free(attempt); + return APPLICATION_ATTEMPT_DECISION_STOP; + } if (disposition == CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL) { execution->unsafe_terminal = true; execution->supervision_failed = diff --git a/src/foundation/index_policy.c b/src/foundation/index_policy.c index 0d2b986bf2..80aa2f8a65 100644 --- a/src/foundation/index_policy.c +++ b/src/foundation/index_policy.c @@ -1,21 +1,38 @@ #include "foundation/index_policy.h" +#include #include #include -static const char *const INDEX_POLICY_KEYS[] = { - CBM_INDEX_CONFIG_MAX_FILES, - CBM_INDEX_CONFIG_MAX_SOURCE_MB, +typedef struct { + const char *key; + size_t field_offset; + uint64_t minimum; + uint64_t maximum; + uint64_t multiplier; +} index_policy_metadata_t; + +static const index_policy_metadata_t INDEX_POLICY_METADATA[] = { + {CBM_INDEX_CONFIG_MAX_FILES, offsetof(cbm_index_resource_policy_t, max_files), 1, + CBM_INDEX_MAX_FILES_VALUE, 1}, + {CBM_INDEX_CONFIG_MAX_SOURCE_MB, offsetof(cbm_index_resource_policy_t, max_source_bytes), 1, + CBM_INDEX_MAX_SOURCE_MB_VALUE, CBM_INDEX_MIB_BYTES}, + {CBM_INDEX_CONFIG_MAX_RSS_MB, offsetof(cbm_index_resource_policy_t, max_rss_bytes), + CBM_INDEX_MIN_RSS_MB_VALUE, CBM_INDEX_MAX_RSS_MB_VALUE, CBM_INDEX_MIB_BYTES}, + {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, offsetof(cbm_index_resource_policy_t, max_duration_ms), + 1, CBM_INDEX_MAX_DURATION_SECONDS_VALUE, UINT64_C(1000)}, }; -static void set_error(char *error, size_t error_size, const char *key, uint64_t maximum) { +static void set_error(char *error, size_t error_size, const char *key, uint64_t minimum, + uint64_t maximum) { if (error && error_size > 0) { - (void)snprintf(error, error_size, "%s must be off or an integer from 1 to %llu", key, - (unsigned long long)maximum); + (void)snprintf(error, error_size, "%s must be off or an integer from %llu to %llu", key, + (unsigned long long)minimum, (unsigned long long)maximum); } } -static bool parse_bounded_uint64(const char *value, uint64_t maximum, uint64_t *parsed) { +static bool parse_bounded_uint64(const char *value, uint64_t minimum, uint64_t maximum, + uint64_t *parsed) { if (!value || !value[0] || !parsed) { return false; } @@ -30,7 +47,7 @@ static bool parse_bounded_uint64(const char *value, uint64_t maximum, uint64_t * } result = result * 10U + digit; } - if (result == 0) { + if (result < minimum) { return false; } *parsed = result; @@ -44,20 +61,39 @@ void cbm_index_policy_init(cbm_index_resource_policy_t *policy) { } bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy) { + if (!policy) { + return false; + } + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + const cbm_index_limit_u64_t *limit = + (const cbm_index_limit_u64_t *)((const unsigned char *)policy + + INDEX_POLICY_METADATA[index].field_offset); + if (limit->enabled) { + return true; + } + } + return false; +} + +bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy) { return policy && (policy->max_files.enabled || policy->max_source_bytes.enabled); } +bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy) { + return policy && (policy->max_rss_bytes.enabled || policy->max_duration_ms.enabled); +} + size_t cbm_index_policy_key_count(void) { - return sizeof(INDEX_POLICY_KEYS) / sizeof(INDEX_POLICY_KEYS[0]); + return sizeof(INDEX_POLICY_METADATA) / sizeof(INDEX_POLICY_METADATA[0]); } const char *cbm_index_policy_key_at(size_t index) { - return index < cbm_index_policy_key_count() ? INDEX_POLICY_KEYS[index] : NULL; + return index < cbm_index_policy_key_count() ? INDEX_POLICY_METADATA[index].key : NULL; } const char *cbm_index_policy_default_value(const char *key) { for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { - if (key && strcmp(key, INDEX_POLICY_KEYS[index]) == 0) { + if (key && strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { return "off"; } } @@ -70,34 +106,33 @@ bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, error[0] = '\0'; } if (!policy || !key || !value) { - set_error(error, error_size, key ? key : "index resource limit", 0); + set_error(error, error_size, key ? key : "index resource limit", 0, 0); return false; } - cbm_index_limit_u64_t *target = NULL; - uint64_t maximum = 0; - uint64_t multiplier = 1; - if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) { - target = &policy->max_files; - maximum = CBM_INDEX_MAX_FILES_VALUE; - } else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) { - target = &policy->max_source_bytes; - maximum = CBM_INDEX_MAX_SOURCE_MB_VALUE; - multiplier = CBM_INDEX_MIB_BYTES; - } else { - set_error(error, error_size, key, 0); + const index_policy_metadata_t *metadata = NULL; + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { + metadata = &INDEX_POLICY_METADATA[index]; + break; + } + } + if (!metadata) { + set_error(error, error_size, key, 0, 0); return false; } + cbm_index_limit_u64_t *target = + (cbm_index_limit_u64_t *)((unsigned char *)policy + metadata->field_offset); cbm_index_limit_u64_t candidate = {0}; if (strcmp(value, "off") != 0) { uint64_t parsed = 0; - if (!parse_bounded_uint64(value, maximum, &parsed)) { - set_error(error, error_size, key, maximum); + if (!parse_bounded_uint64(value, metadata->minimum, metadata->maximum, &parsed)) { + set_error(error, error_size, key, metadata->minimum, metadata->maximum); return false; } candidate.enabled = true; - candidate.value = parsed * multiplier; + candidate.value = parsed * metadata->multiplier; } *target = candidate; return true; @@ -108,18 +143,21 @@ bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const ch if (!policy || !key || !out || out_size == 0) { return false; } - const cbm_index_limit_u64_t *limit = NULL; - uint64_t divisor = 1; - if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) { - limit = &policy->max_files; - } else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) { - limit = &policy->max_source_bytes; - divisor = CBM_INDEX_MIB_BYTES; - } else { + const index_policy_metadata_t *metadata = NULL; + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { + metadata = &INDEX_POLICY_METADATA[index]; + break; + } + } + if (!metadata) { return false; } + const cbm_index_limit_u64_t *limit = + (const cbm_index_limit_u64_t *)((const unsigned char *)policy + metadata->field_offset); int length = limit->enabled - ? snprintf(out, out_size, "%llu", (unsigned long long)(limit->value / divisor)) + ? snprintf(out, out_size, "%llu", + (unsigned long long)(limit->value / metadata->multiplier)) : snprintf(out, out_size, "off"); return length >= 0 && (size_t)length < out_size; } @@ -130,6 +168,10 @@ const char *cbm_index_resource_name(cbm_index_resource_t resource) { return "files"; case CBM_INDEX_RESOURCE_SOURCE_BYTES: return "source_bytes"; + case CBM_INDEX_RESOURCE_RSS_BYTES: + return "rss_bytes"; + case CBM_INDEX_RESOURCE_DURATION_MS: + return "duration_ms"; case CBM_INDEX_RESOURCE_NONE: default: return "unknown"; @@ -137,7 +179,13 @@ const char *cbm_index_resource_name(cbm_index_resource_t resource) { } const char *cbm_index_resource_unit(cbm_index_resource_t resource) { - return resource == CBM_INDEX_RESOURCE_FILES ? "files" : "bytes"; + if (resource == CBM_INDEX_RESOURCE_FILES) { + return "files"; + } + if (resource == CBM_INDEX_RESOURCE_DURATION_MS) { + return "milliseconds"; + } + return "bytes"; } const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { @@ -146,6 +194,10 @@ const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { return CBM_INDEX_CONFIG_MAX_FILES; case CBM_INDEX_RESOURCE_SOURCE_BYTES: return CBM_INDEX_CONFIG_MAX_SOURCE_MB; + case CBM_INDEX_RESOURCE_RSS_BYTES: + return CBM_INDEX_CONFIG_MAX_RSS_MB; + case CBM_INDEX_RESOURCE_DURATION_MS: + return CBM_INDEX_CONFIG_MAX_DURATION_SECONDS; case CBM_INDEX_RESOURCE_NONE: default: return "index_resource_limit"; diff --git a/src/foundation/index_policy.h b/src/foundation/index_policy.h index 6a6c499fc6..617a6ab7de 100644 --- a/src/foundation/index_policy.h +++ b/src/foundation/index_policy.h @@ -7,9 +7,14 @@ #define CBM_INDEX_CONFIG_MAX_FILES "index_max_files" #define CBM_INDEX_CONFIG_MAX_SOURCE_MB "index_max_source_mb" +#define CBM_INDEX_CONFIG_MAX_RSS_MB "index_max_rss_mb" +#define CBM_INDEX_CONFIG_MAX_DURATION_SECONDS "index_max_duration_seconds" #define CBM_INDEX_MAX_FILES_VALUE UINT64_C(10000000) #define CBM_INDEX_MAX_SOURCE_MB_VALUE UINT64_C(1048576) +#define CBM_INDEX_MIN_RSS_MB_VALUE UINT64_C(64) +#define CBM_INDEX_MAX_RSS_MB_VALUE UINT64_C(1048576) +#define CBM_INDEX_MAX_DURATION_SECONDS_VALUE UINT64_C(86400) #define CBM_INDEX_MIB_BYTES UINT64_C(1048576) typedef struct { @@ -20,12 +25,16 @@ typedef struct { typedef struct { cbm_index_limit_u64_t max_files; cbm_index_limit_u64_t max_source_bytes; + cbm_index_limit_u64_t max_rss_bytes; + cbm_index_limit_u64_t max_duration_ms; } cbm_index_resource_policy_t; typedef enum { CBM_INDEX_RESOURCE_NONE = 0, CBM_INDEX_RESOURCE_FILES, CBM_INDEX_RESOURCE_SOURCE_BYTES, + CBM_INDEX_RESOURCE_RSS_BYTES, + CBM_INDEX_RESOURCE_DURATION_MS, } cbm_index_resource_t; typedef struct { @@ -36,6 +45,8 @@ typedef struct { void cbm_index_policy_init(cbm_index_resource_policy_t *policy); bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy); size_t cbm_index_policy_key_count(void); const char *cbm_index_policy_key_at(size_t index); diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 5e3a549ed2..474ada1d2b 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -14,6 +14,7 @@ #include "platform.h" /* cbm_now_ms */ #include "sanitized.h" /* CBM_SANITIZED — spawn-retry budget */ +#include #include #include #include @@ -21,15 +22,18 @@ #ifdef _WIN32 #include +#include #include "win_utf8.h" /* cbm_utf8_to_wide — spawn the worker with a wide command line so a * non-ASCII repo path survives CreateProcess (#423/#20) */ #include /* free */ #else +#include #include #include #include #ifdef __APPLE__ #include +#include #include #endif extern char **environ; @@ -456,6 +460,287 @@ struct cbm_subprocess { #endif }; +static void cbm_rss_add_saturated(uint64_t *total, uint64_t value) { + if (value > UINT64_MAX - *total) { + *total = UINT64_MAX; + } else { + *total += value; + } +} + +#ifdef CBM_ENABLE_TEST_SEAMS +uint64_t cbm_subprocess_rss_sum_for_testing(const uint64_t *values, size_t count) { + uint64_t total = 0; + for (size_t index = 0; values && index < count; index++) { + cbm_rss_add_saturated(&total, values[index]); + } + return total; +} +#endif + +#ifdef _WIN32 +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + DWORD capacity = 32; + JOBOBJECT_BASIC_PROCESS_ID_LIST *processes = NULL; + for (;;) { + size_t bytes = sizeof(*processes) + (size_t)(capacity - 1) * sizeof(ULONG_PTR); + if (bytes > UINT32_MAX) { + cbm_free(CBM_MEM_CLASS_OTHER, processes); + return CBM_PROC_TREE_RSS_ERROR; + } + JOBOBJECT_BASIC_PROCESS_ID_LIST *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, processes, bytes); + if (!grown) { + cbm_free(CBM_MEM_CLASS_OTHER, processes); + return CBM_PROC_TREE_RSS_ERROR; + } + processes = grown; + ZeroMemory(processes, bytes); + processes->NumberOfAssignedProcesses = capacity; + if (QueryInformationJobObject(process->job, JobObjectBasicProcessIdList, processes, + (DWORD)bytes, NULL)) { + break; + } + if (GetLastError() != ERROR_MORE_DATA || capacity > (UINT32_MAX / 2U)) { + cbm_free(CBM_MEM_CLASS_OTHER, processes); + return CBM_PROC_TREE_RSS_ERROR; + } + capacity *= 2U; + } + + uint64_t total = 0; + DWORD measured = 0; + bool root_failed = false; + for (DWORD index = 0; index < processes->NumberOfProcessIdsInList; index++) { + DWORD pid = (DWORD)processes->ProcessIdList[index]; + HANDLE member = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid); + if (!member) { + root_failed = root_failed || pid == process->process_id; + continue; + } + PROCESS_MEMORY_COUNTERS memory; + ZeroMemory(&memory, sizeof(memory)); + if (GetProcessMemoryInfo(member, &memory, sizeof(memory))) { + cbm_rss_add_saturated(&total, (uint64_t)memory.WorkingSetSize); + measured++; + } else if (pid == process->process_id) { + root_failed = true; + } + CloseHandle(member); + } + DWORD listed = processes->NumberOfProcessIdsInList; + cbm_free(CBM_MEM_CLASS_OTHER, processes); + if (root_failed || (measured == 0 && listed > 0)) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#elif defined(__APPLE__) +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + int capacity = proc_listallpids(NULL, 0); + if (capacity <= 0 || capacity > INT_MAX / (int)sizeof(pid_t) - 64) { + return CBM_PROC_TREE_RSS_ERROR; + } + capacity += 64; + pid_t *pids = NULL; + int count = 0; + for (;;) { + pid_t *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, pids, (size_t)capacity * sizeof(*pids)); + if (!grown) { + cbm_free(CBM_MEM_CLASS_OTHER, pids); + return CBM_PROC_TREE_RSS_ERROR; + } + pids = grown; + count = proc_listallpids(pids, capacity * (int)sizeof(*pids)); + if (count <= 0) { + cbm_free(CBM_MEM_CLASS_OTHER, pids); + return CBM_PROC_TREE_RSS_ERROR; + } + if (count < capacity) { + break; + } + if (capacity > INT_MAX / (int)sizeof(*pids) / 2) { + cbm_free(CBM_MEM_CLASS_OTHER, pids); + return CBM_PROC_TREE_RSS_ERROR; + } + capacity *= 2; + } + + uint64_t total = 0; + int measured = 0; + bool root_failed = false; + for (int index = 0; index < count; index++) { + struct proc_bsdinfo info; + if (pids[index] <= 0) { + continue; + } + if (proc_pidinfo(pids[index], PROC_PIDTBSDINFO, 0, &info, sizeof(info)) != + (int)sizeof(info)) { + root_failed = root_failed || pids[index] == process->pid; + continue; + } + if ((pid_t)info.pbi_pgid != process->pgid) { + continue; + } + struct rusage_info_v2 usage; + if (proc_pid_rusage(pids[index], RUSAGE_INFO_V2, (rusage_info_t *)&usage) != 0) { + root_failed = root_failed || pids[index] == process->pid; + continue; + } + cbm_rss_add_saturated(&total, usage.ri_resident_size); + measured++; + } + cbm_free(CBM_MEM_CLASS_OTHER, pids); + if (root_failed) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#else +static bool cbm_proc_pid_name(const char *name) { + if (!name || !name[0]) { + return false; + } + for (const unsigned char *cursor = (const unsigned char *)name; *cursor; cursor++) { + if (*cursor < '0' || *cursor > '9') { + return false; + } + } + return true; +} + +static bool cbm_linux_proc_stat(const char *pid_name, pid_t *group, int64_t *rss_pages) { + char path[64]; + int length = snprintf(path, sizeof(path), "/proc/%s/stat", pid_name); + if (length <= 0 || length >= (int)sizeof(path)) { + return false; + } + FILE *stat_file = cbm_fopen(path, "rb"); + if (!stat_file) { + return false; + } + char stat_line[4096]; + bool read = fgets(stat_line, sizeof(stat_line), stat_file) != NULL; + (void)fclose(stat_file); + char *command_end = read ? strrchr(stat_line, ')') : NULL; + if (!command_end || command_end[1] != ' ') { + return false; + } + + char *save = NULL; + char *token = strtok_r(command_end + 2, " ", &save); + int field = 3; + bool have_group = false; + bool have_rss = false; + while (token) { + if (field == 5 || field == 24) { + char *end = NULL; + long long value = strtoll(token, &end, 10); + if (!end || *end != '\0') { + return false; + } + if (field == 5) { + *group = (pid_t)value; + have_group = true; + } else { + *rss_pages = (int64_t)value; + have_rss = true; + break; + } + } + token = strtok_r(NULL, " ", &save); + field++; + } + return have_group && have_rss; +} + +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + DIR *proc = opendir("/proc"); + if (!proc) { + return CBM_PROC_TREE_RSS_ERROR; + } + long page_size = sysconf(_SC_PAGESIZE); + if (page_size <= 0) { + (void)closedir(proc); + return CBM_PROC_TREE_RSS_ERROR; + } + + uint64_t total = 0; + int measured = 0; + bool root_failed = false; + struct dirent *entry; + while ((entry = readdir(proc)) != NULL) { + if (!cbm_proc_pid_name(entry->d_name)) { + continue; + } + pid_t group = 0; + int64_t pages = 0; + bool root_entry = strtol(entry->d_name, NULL, 10) == (long)process->pid; + if (!cbm_linux_proc_stat(entry->d_name, &group, &pages)) { + root_failed = root_failed || root_entry; + continue; + } + if (group != process->pgid || pages < 0) { + continue; + } + uint64_t page_count = (uint64_t)pages; + uint64_t bytes = page_count > UINT64_MAX / (uint64_t)page_size + ? UINT64_MAX + : page_count * (uint64_t)page_size; + cbm_rss_add_saturated(&total, bytes); + measured++; + } + (void)closedir(proc); + if (root_failed) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#endif + +cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_bytes(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + if (!process || !rss_bytes) { + return CBM_PROC_TREE_RSS_ERROR; + } + *rss_bytes = 0; + return cbm_subprocess_tree_rss_platform(process, rss_bytes); +} + +bool cbm_subprocess_root_running(const cbm_subprocess_t *process) { + if (!process || process->root_reaped) { + return false; + } + int lifecycle = atomic_load_explicit(&process->lifecycle, memory_order_acquire); + return lifecycle == CBM_SUBPROCESS_ACTIVE || lifecycle == CBM_SUBPROCESS_CANCEL_REQUESTED; +} + +bool cbm_subprocess_termination_pending(const cbm_subprocess_t *process) { + return process && process->termination_started; +} + +bool cbm_subprocess_supervision_active(const cbm_subprocess_t *process) { + if (!process) { + return false; + } + int lifecycle = atomic_load_explicit(&process->lifecycle, memory_order_acquire); + return lifecycle == CBM_SUBPROCESS_ACTIVE || lifecycle == CBM_SUBPROCESS_CANCEL_REQUESTED; +} + static void cbm_subprocess_result_init(cbm_proc_result_t *result) { result->outcome = CBM_PROC_SPAWN_FAILED; result->exit_code = -1; diff --git a/src/foundation/subprocess.h b/src/foundation/subprocess.h index 4dc8f18705..bfcb2311b7 100644 --- a/src/foundation/subprocess.h +++ b/src/foundation/subprocess.h @@ -22,6 +22,7 @@ #include #include /* size_t (cbm_build_win_cmdline) */ +#include /* How a supervised child ended. */ typedef enum { @@ -118,6 +119,12 @@ typedef enum { CBM_PROC_POLL_TERMINAL = 1 } cbm_proc_poll_t; +typedef enum { + CBM_PROC_TREE_RSS_ERROR = -1, + CBM_PROC_TREE_RSS_EMPTY = 0, + CBM_PROC_TREE_RSS_OK = 1, +} cbm_proc_tree_rss_status_t; + /* Spawn opts->bin and return immediately with a supervisor handle. On success, * *out owns the process until a terminal poll followed by destroy. Spawn copies * the option strings/argv it needs after return; log_ud remains caller-owned until @@ -148,6 +155,16 @@ int cbm_subprocess_spawn(const cbm_proc_opts_t *opts, cbm_subprocess_t **out); * tree when cancel_grace_ms elapses. Callers must keep polling to make progress. */ cbm_proc_poll_t cbm_subprocess_poll(cbm_subprocess_t *process, cbm_proc_result_t *out); +/* Read the current resident-set size of the complete contained process tree. + * OK returns an overflow-safe byte total, EMPTY means the owned tree currently + * has no observable members, and ERROR means no trustworthy measurement could + * be obtained. Individual processes that exit during enumeration are ignored. */ +cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_bytes(cbm_subprocess_t *process, + uint64_t *rss_bytes); +bool cbm_subprocess_root_running(const cbm_subprocess_t *process); +bool cbm_subprocess_termination_pending(const cbm_subprocess_t *process); +bool cbm_subprocess_supervision_active(const cbm_subprocess_t *process); + /* Record an explicit cancellation request without waiting. Safe to repeat and * safe to call from a cancellation thread while one owner thread polls. The * owner must stop cancellation producers before destroying the handle. true @@ -223,6 +240,7 @@ void cbm_subprocess_hold_parent_until_child_exits_for_testing(int spawns); * shows for a child that is EXITING: setpgid and getpgid answer ESRCH while * waitid does not report it yet. Test builds only. */ void cbm_subprocess_observe_exiting_child_for_testing(int spawns); +uint64_t cbm_subprocess_rss_sum_for_testing(const uint64_t *values, size_t count); #endif /* How the POSIX fork+exec child closed its inherited descriptors (#1484). */ diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index 223945d3c3..4ffeecbb89 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -481,8 +481,56 @@ enum { INDEX_WORKER_SYNC_POLL_NS = 10000000, INDEX_WORKER_RELAY_LINES_PER_POLL = 64, INDEX_WORKER_RELAY_BYTES_PER_POLL = 64 * 1024, + INDEX_WORKER_RSS_PROBE_FAILURE_LIMIT = 3, + INDEX_WORKER_RSS_PROBE_INTERVAL_MS = 250, }; +typedef enum { + INDEX_WORKER_TERMINATION_NONE = 0, + INDEX_WORKER_TERMINATION_CANCEL_PENDING, + INDEX_WORKER_TERMINATION_CANCEL, + INDEX_WORKER_TERMINATION_RESOURCE, +} index_worker_termination_reason_t; + +#ifdef CBM_ENABLE_TEST_SEAMS +static cbm_index_supervisor_clock_fn g_resource_clock; +static cbm_index_supervisor_rss_fn g_resource_rss; +static void *g_resource_hook_context; + +void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_clock_fn clock_fn, + cbm_index_supervisor_rss_fn rss_fn, + void *context) { + g_resource_clock = clock_fn; + g_resource_rss = rss_fn; + g_resource_hook_context = context; +} + +void cbm_index_supervisor_reset_resource_hooks_for_testing(void) { + g_resource_clock = NULL; + g_resource_rss = NULL; + g_resource_hook_context = NULL; +} +#endif + +static uint64_t worker_resource_now_ms(void) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_resource_clock) { + return g_resource_clock(g_resource_hook_context); + } +#endif + return cbm_now_ms(); +} + +static cbm_proc_tree_rss_status_t worker_resource_rss(cbm_subprocess_t *process, + uint64_t *rss_bytes) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_resource_rss) { + return g_resource_rss(process, rss_bytes, g_resource_hook_context); + } +#endif + return cbm_subprocess_tree_rss_bytes(process, rss_bytes); +} + struct cbm_index_worker_handle { cbm_subprocess_t *process; char response_path[INDEX_WORKER_PATH_CAP]; @@ -492,6 +540,12 @@ struct cbm_index_worker_handle { long relay_tail_pos; bool process_terminal; cbm_proc_result_t process_result; + cbm_index_resource_policy_t resource_policy; + uint64_t started_ms; + uint64_t last_rss_probe_ms; + unsigned int rss_probe_failures; + bool rss_probe_started; + atomic_int termination_reason; atomic_bool terminal; cbm_index_worker_result_t result; }; @@ -670,7 +724,8 @@ static bool worker_unique_file(char *out, size_t out_size, const char *kind) { static bool worker_result_succeeded(const cbm_index_worker_result_t *result) { return result && result->outcome == CBM_PROC_CLEAN && !result->cancellation_requested && - result->tree_quiesced && !result->supervision_failed; + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed && result->tree_quiesced && !result->supervision_failed; } static void worker_terminal_log(cbm_index_worker_handle_t *handle) { @@ -701,6 +756,13 @@ static void worker_terminal_log(cbm_index_worker_handle_t *handle) { } else if (handle->result.supervision_failed || !handle->result.tree_quiesced) { cbm_log_error("index.supervisor.containment_failed", "outcome", cbm_proc_outcome_str(handle->result.outcome), "log", handle->log_path); + } else if (handle->result.resource_probe_failed) { + cbm_log_error("index.supervisor.resource_probe_failed", "resource", "rss_bytes", "log", + handle->log_path); + } else if (handle->result.resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { + cbm_log_warn("index.supervisor.resource_limit", "resource", + cbm_index_resource_name(handle->result.resource_violation.resource), "log", + handle->log_path); } else if (handle->result.cancellation_requested) { cbm_log_warn("index.supervisor.worker_cancelled", "outcome", cbm_proc_outcome_str(handle->result.outcome), "log", handle->log_path); @@ -739,10 +801,87 @@ size_t cbm_index_worker_job_memory_limit(size_t memory_budget_bytes) { return memory_budget_bytes > SIZE_MAX - headroom ? SIZE_MAX : memory_budget_bytes + headroom; } -int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, - bool single_thread, const char *marker_file, - const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, cbm_index_worker_handle_t **handle_out) { +static void worker_request_resource_termination(cbm_index_worker_handle_t *handle, + cbm_index_resource_t resource, uint64_t observed, + uint64_t limit, bool probe_failed) { + int expected = INDEX_WORKER_TERMINATION_NONE; + if (!atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + INDEX_WORKER_TERMINATION_RESOURCE, + memory_order_acq_rel, memory_order_acquire)) { + return; + } + handle->result.resource_violation = (cbm_index_resource_violation_t){ + .resource = resource, .observed = observed, .limit = limit}; + handle->result.resource_probe_failed = probe_failed; + if (!cbm_subprocess_request_cancel(handle->process)) { + handle->result.resource_violation = (cbm_index_resource_violation_t){0}; + handle->result.resource_probe_failed = false; + expected = INDEX_WORKER_TERMINATION_RESOURCE; + (void)atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + INDEX_WORKER_TERMINATION_NONE, + memory_order_acq_rel, memory_order_acquire); + } +} + +static bool worker_check_resource_limits(cbm_index_worker_handle_t *handle) { + if (atomic_load_explicit(&handle->termination_reason, memory_order_acquire) != + INDEX_WORKER_TERMINATION_NONE || + cbm_subprocess_termination_pending(handle->process) || + !cbm_subprocess_supervision_active(handle->process)) { + return false; + } + uint64_t now = 0; + bool have_now = false; + bool rss_probe_failed = false; + if (handle->resource_policy.max_rss_bytes.enabled) { + now = worker_resource_now_ms(); + have_now = true; + bool probe_due = !handle->rss_probe_started || now < handle->last_rss_probe_ms || + now - handle->last_rss_probe_ms >= INDEX_WORKER_RSS_PROBE_INTERVAL_MS; + if (!probe_due) { + goto duration_check; + } + handle->rss_probe_started = true; + handle->last_rss_probe_ms = now; + uint64_t rss_bytes = 0; + cbm_proc_tree_rss_status_t status = worker_resource_rss(handle->process, &rss_bytes); + if (status == CBM_PROC_TREE_RSS_OK) { + handle->rss_probe_failures = 0; + if (rss_bytes > handle->resource_policy.max_rss_bytes.value) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_RSS_BYTES, rss_bytes, + handle->resource_policy.max_rss_bytes.value, + false); + return false; + } + } else if (status == CBM_PROC_TREE_RSS_ERROR) { + handle->rss_probe_failures++; + if (handle->rss_probe_failures >= INDEX_WORKER_RSS_PROBE_FAILURE_LIMIT) { + rss_probe_failed = true; + } + } else { + handle->rss_probe_failures = 0; + } + } +duration_check: + if (handle->resource_policy.max_duration_ms.enabled) { + if (!have_now) { + now = worker_resource_now_ms(); + } + uint64_t elapsed = now >= handle->started_ms ? now - handle->started_ms : 0; + if (elapsed > handle->resource_policy.max_duration_ms.value) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_DURATION_MS, elapsed, + handle->resource_policy.max_duration_ms.value, + false); + } + } + return rss_probe_failed; +} + +static int worker_start_internal(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, cbm_index_worker_handle_t **handle_out) { if (handle_out) { *handle_out = NULL; } @@ -774,6 +913,11 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ if (!handle) { return -1; } + cbm_index_policy_init(&handle->resource_policy); + if (resource_policy) { + handle->resource_policy = *resource_policy; + } + atomic_init(&handle->termination_reason, INDEX_WORKER_TERMINATION_NONE); atomic_init(&handle->terminal, false); handle->log_callback = log_callback; handle->log_context = log_context; @@ -840,15 +984,46 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ cbm_log_error("index.supervisor.spawn_failed", "action", "fail_closed"); return -1; } + if (handle->resource_policy.max_duration_ms.enabled) { + handle->started_ms = worker_resource_now_ms(); + } *handle_out = handle; return 0; } +int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, + bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, + quarantine_file, log_callback, log_context, handle_out); +} + int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out) { - return cbm_index_worker_start_with_log(args_json, memory_budget_bytes, single_thread, - marker_file, quarantine_file, NULL, NULL, handle_out); + return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, + quarantine_file, NULL, NULL, handle_out); +} + +int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, + cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, + marker_file, quarantine_file, NULL, NULL, handle_out); +} + +int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, + marker_file, quarantine_file, log_callback, log_context, + handle_out); } cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, @@ -865,10 +1040,16 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, } bool relay_caught_up = true; if (!handle->process_terminal) { + bool rss_probe_failed = worker_check_resource_limits(handle); cbm_proc_result_t process_result; cbm_proc_poll_t state = cbm_subprocess_poll(handle->process, &process_result); relay_caught_up = worker_relay_log(handle); if (state == CBM_PROC_POLL_RUNNING) { + if (rss_probe_failed && cbm_subprocess_root_running(handle->process)) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_RSS_BYTES, 0, + handle->resource_policy.max_rss_bytes.value, + true); + } return CBM_INDEX_WORKER_POLL_RUNNING; } if (state != CBM_PROC_POLL_TERMINAL) { @@ -889,7 +1070,12 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, handle->result.outcome = process_result->outcome; handle->result.exit_code = process_result->exit_code; handle->result.term_signal = process_result->term_signal; - handle->result.cancellation_requested = process_result->cancellation_requested; + int termination_reason = + atomic_load_explicit(&handle->termination_reason, memory_order_acquire); + handle->result.cancellation_requested = + termination_reason == INDEX_WORKER_TERMINATION_CANCEL || + (termination_reason == INDEX_WORKER_TERMINATION_CANCEL_PENDING && + process_result->cancellation_requested); handle->result.forced = process_result->forced; handle->result.tree_quiesced = process_result->tree_quiesced; handle->result.supervision_failed = process_result->supervision_failed; @@ -922,8 +1108,36 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, } bool cbm_index_worker_request_cancel(cbm_index_worker_handle_t *handle) { - return handle && !atomic_load_explicit(&handle->terminal, memory_order_acquire) && - cbm_subprocess_request_cancel(handle->process); + if (!handle || atomic_load_explicit(&handle->terminal, memory_order_acquire)) { + return false; + } + int expected = INDEX_WORKER_TERMINATION_NONE; + bool claimed = atomic_compare_exchange_strong_explicit( + &handle->termination_reason, &expected, INDEX_WORKER_TERMINATION_CANCEL_PENDING, + memory_order_acq_rel, memory_order_acquire); + if (!claimed) { + if (expected == INDEX_WORKER_TERMINATION_CANCEL) { + return true; + } + if (expected != INDEX_WORKER_TERMINATION_CANCEL_PENDING) { + return false; + } + bool accepted = cbm_subprocess_request_cancel(handle->process); + if (accepted) { + expected = INDEX_WORKER_TERMINATION_CANCEL_PENDING; + (void)atomic_compare_exchange_strong_explicit( + &handle->termination_reason, &expected, INDEX_WORKER_TERMINATION_CANCEL, + memory_order_acq_rel, memory_order_acquire); + } + return accepted; + } + bool accepted = cbm_subprocess_request_cancel(handle->process); + expected = INDEX_WORKER_TERMINATION_CANCEL_PENDING; + (void)atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + accepted ? INDEX_WORKER_TERMINATION_CANCEL + : INDEX_WORKER_TERMINATION_NONE, + memory_order_acq_rel, memory_order_acquire); + return accepted; } const char *cbm_index_worker_response_path(const cbm_index_worker_handle_t *handle) { @@ -948,18 +1162,19 @@ void cbm_index_worker_destroy(cbm_index_worker_handle_t *handle) { free(handle); } -int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_thread, - const char *marker_file, const char *quarantine_file, - cbm_proc_log_cb log_callback, void *log_context, - const atomic_int *cancel_requested, - cbm_index_worker_result_t *result) { +static int worker_spawn_internal(const char *args_json, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, + cbm_index_worker_result_t *result) { if (!result) { return -1; } worker_result_init(result); cbm_index_worker_handle_t *handle = NULL; - if (cbm_index_worker_start_with_log(args_json, 0, single_thread, marker_file, quarantine_file, - log_callback, log_context, &handle) != 0) { + if (worker_start_internal(args_json, 0, resource_policy, single_thread, marker_file, + quarantine_file, log_callback, log_context, &handle) != 0) { return -1; } const cbm_index_worker_result_t *cached = NULL; @@ -985,6 +1200,24 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th return 0; } +int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + const atomic_int *cancel_requested, + cbm_index_worker_result_t *result) { + return worker_spawn_internal(args_json, NULL, single_thread, marker_file, quarantine_file, + log_callback, log_context, cancel_requested, result); +} + +int cbm_index_spawn_worker_with_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, + const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { + return worker_spawn_internal(args_json, resource_policy, single_thread, marker_file, + quarantine_file, log_callback, log_context, cancel_requested, + result); +} + int cbm_index_spawn_worker_with_log(const char *args_json, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, diff --git a/src/mcp/index_supervisor.h b/src/mcp/index_supervisor.h index c231a563e3..a86761d773 100644 --- a/src/mcp/index_supervisor.h +++ b/src/mcp/index_supervisor.h @@ -22,8 +22,10 @@ #include #include +#include #include +#include "foundation/index_policy.h" #include "foundation/subprocess.h" /* cbm_proc_outcome_t */ /* Worker-role state, set once from the CLI arg parser (main.c) when this process @@ -156,8 +158,10 @@ typedef struct { char last_phase[128]; char worker_log[4096]; bool response_rejected; /* clean worker exceeded the bounded response protocol */ - char *response; /* worker result only after a contained, uncancelled CLEAN exit; - * borrowed for async polls, caller-owned from the sync wrapper */ + cbm_index_resource_violation_t resource_violation; + bool resource_probe_failed; + char *response; /* worker result only after a contained, uncancelled CLEAN exit; + * borrowed for async polls, caller-owned from the sync wrapper */ } cbm_index_worker_result_t; /* Daemon-owned, nonblocking supervisor for one contained worker process tree. */ @@ -179,6 +183,21 @@ size_t cbm_index_worker_job_memory_limit(size_t memory_budget_bytes); int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out); +int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, + cbm_index_worker_handle_t **handle_out); + +/* Policy plus the request-scoped log callback. The daemon default start path + * needs both: a missing policy fails closed, and worker diagnostics still + * reach the daemon journal. */ +int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + cbm_index_worker_handle_t **handle_out); /* Request-scoped variant used by interactive local CLI calls. The callback is * invoked by the owner thread while it polls the contained worker; log_context @@ -242,7 +261,22 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th cbm_proc_log_cb log_callback, void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result); +int cbm_index_spawn_worker_with_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, + const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result); void cbm_index_worker_result_free(cbm_index_worker_result_t *result); +#ifdef CBM_ENABLE_TEST_SEAMS +typedef uint64_t (*cbm_index_supervisor_clock_fn)(void *context); +typedef cbm_proc_tree_rss_status_t (*cbm_index_supervisor_rss_fn)(cbm_subprocess_t *process, + uint64_t *rss_bytes, + void *context); +void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_clock_fn clock_fn, + cbm_index_supervisor_rss_fn rss_fn, + void *context); +void cbm_index_supervisor_reset_resource_hooks_for_testing(void); +#endif + #endif /* CBM_INDEX_SUPERVISOR_H */ diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 1d1489e7fc..1016f781ab 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -11205,6 +11205,22 @@ static bool build_index_success_response(cbm_mcp_server_t *srv, yyjson_mut_doc * return degraded; } +static bool project_db_is_servable(const char *project, const char *db_path); + +/* Serialize a worker JSON document and release the heap strings it borrowed. + * Extra arguments may be NULL; free(NULL) is a no-op. */ +static char *index_worker_json_result(yyjson_mut_doc *doc, char *repo_path, char *name_override, + char *project_name) { + char *json = yy_doc_to_str(doc); + yyjson_mut_doc_free(doc); + free(repo_path); + free(name_override); + free(project_name); + char *result = cbm_mcp_text_result(json, true); + free(json); + return result; +} + /* Build the response for a worker that crashed/hung/failed without producing a * result. The crash is already contained (this process survived); we report it * rather than dying. Precise skip-and-continue (quarantine the culprit, index the @@ -11237,12 +11253,7 @@ static char *build_worker_failure_response(const char *args, cbm_proc_outcome_t if (repo_path) { yyjson_mut_obj_add_strcpy(doc, root, "repo_path", repo_path); } - char *json = yy_doc_to_str(doc); - yyjson_mut_doc_free(doc); - free(repo_path); - char *result = cbm_mcp_text_result(json, true); - free(json); - return result; + return index_worker_json_result(doc, repo_path, NULL, NULL); } static char *build_worker_unsafe_terminal_response(const char *args, cbm_proc_outcome_t outcome, @@ -11263,12 +11274,50 @@ static char *build_worker_unsafe_terminal_response(const char *args, cbm_proc_ou if (repo_path) { yyjson_mut_obj_add_strcpy(doc, root, "repo_path", repo_path); } - char *json = yy_doc_to_str(doc); - yyjson_mut_doc_free(doc); - free(repo_path); - char *response = cbm_mcp_text_result(json, true); - free(json); - return response; + return index_worker_json_result(doc, repo_path, NULL, NULL); +} + +char *cbm_mcp_index_worker_resource_response(const char *args, + const cbm_index_worker_result_t *worker_result) { + char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); + char *name_override = cbm_mcp_get_string_arg(args, "name"); + char *project_name = + cbm_project_name_from_path(name_override && name_override[0] ? name_override : repo_path); + char db_path[CBM_SZ_1K] = {0}; + if (project_name) { + project_db_path(project_name, db_path, sizeof(db_path)); + } + + const cbm_index_resource_violation_t *violation = &worker_result->resource_violation; + const char *config_key = cbm_index_resource_config_key(violation->resource); + char message[CBM_SZ_256]; + (void)snprintf(message, sizeof(message), + worker_result->resource_probe_failed + ? "Worker resource measurement failed for %s" + : "Index worker exceeded %s", + config_key); + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root); + yyjson_mut_obj_add_str(doc, root, "status", "error"); + yyjson_mut_obj_add_str(doc, root, "code", + worker_result->resource_probe_failed ? "resource_probe_failed" + : "resource_limit_exceeded"); + yyjson_mut_obj_add_str(doc, root, "stage", "worker"); + yyjson_mut_obj_add_str(doc, root, "resource", cbm_index_resource_name(violation->resource)); + if (!worker_result->resource_probe_failed) { + yyjson_mut_obj_add_uint(doc, root, "observed", violation->observed); + yyjson_mut_obj_add_uint(doc, root, "limit", violation->limit); + yyjson_mut_obj_add_str(doc, root, "unit", cbm_index_resource_unit(violation->resource)); + } + yyjson_mut_obj_add_bool(doc, root, "retryable", true); + yyjson_mut_obj_add_bool(doc, root, "serving_index_preserved", + project_name && project_db_is_servable(project_name, db_path)); + yyjson_mut_obj_add_strcpy(doc, root, "message", message); + if (repo_path) { + yyjson_mut_obj_add_strcpy(doc, root, "repo_path", repo_path); + } + return index_worker_json_result(doc, repo_path, name_override, project_name); } /* Drop the cached store so the next query reopens whatever the worker wrote (each @@ -11414,6 +11463,9 @@ cbm_mcp_supervised_result_disposition_t cbm_mcp_supervised_result_disposition( worker_result->supervision_failed) { return CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL; } + if (worker_result->resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { + return CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE; + } if (worker_result->outcome == CBM_PROC_CLEAN) { return worker_result->response ? CBM_MCP_SUPERVISED_RESULT_SUCCESS : CBM_MCP_SUPERVISED_RESULT_FALLBACK; @@ -11421,8 +11473,8 @@ cbm_mcp_supervised_result_disposition_t cbm_mcp_supervised_result_disposition( return CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE; } -static bool index_policy_from_worker_args(const char *args, cbm_index_resource_policy_t *policy, - char *error, size_t error_size) { +bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resource_policy_t *policy, + char *error, size_t error_size) { yyjson_doc *doc = args ? yyjson_read(args, strlen(args), 0) : NULL; yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; yyjson_val *encoded = @@ -11452,7 +11504,7 @@ static bool index_policy_from_worker_args(const char *args, cbm_index_resource_p static bool load_index_policy(cbm_mcp_server_t *srv, const char *args, cbm_index_resource_policy_t *policy, char *error, size_t error_size) { if (cbm_index_worker_active()) { - return index_policy_from_worker_args(args, policy, error, error_size); + return cbm_mcp_index_policy_from_internal_args(args, policy, error, error_size); } cbm_config_t *owned_config = NULL; cbm_config_t *config = srv ? srv->config : NULL; @@ -11562,13 +11614,14 @@ char *cbm_mcp_index_worker_no_response_failure(const char *args, return cbm_mcp_text_result(written ? json : hint, true); } -static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { +static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, + const cbm_index_resource_policy_t *resource_policy) { invalidate_cached_store(srv); /* First attempt: normal parallel run. */ cbm_index_worker_result_t wr; - int rc = cbm_index_spawn_worker_with_log_cancel( - args, false, NULL, NULL, srv ? srv->index_log_callback : NULL, + int rc = cbm_index_spawn_worker_with_policy_log_cancel( + args, resource_policy, false, NULL, NULL, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wr); cbm_mcp_supervised_result_disposition_t disposition = cbm_mcp_supervised_result_disposition(rc, &wr); @@ -11593,6 +11646,12 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { invalidate_cached_store(srv); return failure; } + if (disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + char *failure = cbm_mcp_index_worker_resource_response(args, &wr); + cbm_index_worker_result_free(&wr); + invalidate_cached_store(srv); + return failure; + } if (disposition == CBM_MCP_SUPERVISED_RESULT_SUCCESS) { /* Clean exit → transfer the worker's response (the common path). */ char *resp = wr.response; /* transfer ownership to caller (may be NULL) */ @@ -11642,8 +11701,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { bool terminal_cancelled = false; for (int i = 0; i < cap; i++) { cbm_index_worker_result_t wr2; - int rc2 = cbm_index_spawn_worker_with_log_cancel( - args, /*single_thread=*/false, marker_path, quarantine_path, + int rc2 = cbm_index_spawn_worker_with_policy_log_cancel( + args, resource_policy, /*single_thread=*/false, marker_path, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wr2); cbm_mcp_supervised_result_disposition_t recovery_disposition = @@ -11670,6 +11729,11 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { cbm_index_worker_result_free(&wr2); break; } + if (recovery_disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + resp = cbm_mcp_index_worker_resource_response(args, &wr2); + cbm_index_worker_result_free(&wr2); + break; + } if (recovery_disposition == CBM_MCP_SUPERVISED_RESULT_SUCCESS) { resp = wr2.response; /* transfer ownership to caller */ wr2.response = NULL; @@ -11764,8 +11828,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { * so it cannot itself hang. Rare given monotonic progress. */ if (!resp && !unsafe_terminal && quarantined > 0) { cbm_index_worker_result_t wrp; - int rcp = cbm_index_spawn_worker_with_log_cancel( - args, /*single_thread=*/false, NULL, quarantine_path, + int rcp = cbm_index_spawn_worker_with_policy_log_cancel( + args, resource_policy, /*single_thread=*/false, NULL, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wrp); cbm_mcp_supervised_result_disposition_t partial_disposition = @@ -11781,6 +11845,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { last_outcome = wrp.outcome; unsafe_terminal = true; terminal_cancelled = wrp.cancellation_requested; + } else if (partial_disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + resp = cbm_mcp_index_worker_resource_response(args, &wrp); } cbm_index_worker_result_free(&wrp); } @@ -11827,7 +11893,7 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p if (!args) { return NULL; } - char *resp = index_run_supervised(srv, args); + char *resp = index_run_supervised(srv, args, &policy); free(args); return resp; } @@ -12366,7 +12432,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { index_args_free(repo_path, mode_str, name_override); return cbm_mcp_text_result("failed to prepare supervised index request", true); } - char *supervised = index_run_supervised(srv, worker_args); + char *supervised = index_run_supervised(srv, worker_args, &resource_policy); free(worker_args); if (supervised) { free(mutation_project); diff --git a/src/mcp/mcp.h b/src/mcp/mcp.h index 3a5fce5ff8..a7284672b2 100644 --- a/src/mcp/mcp.h +++ b/src/mcp/mcp.h @@ -275,6 +275,7 @@ char *cbm_mcp_index_worker_no_response_failure(const char *args, typedef enum { CBM_MCP_SUPERVISED_RESULT_FALLBACK = 0, CBM_MCP_SUPERVISED_RESULT_SUCCESS, + CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE, CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE, CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL, } cbm_mcp_supervised_result_disposition_t; diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index 7035d80baf..3ca7c6ee9f 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -60,6 +60,10 @@ bool cbm_mcp_jsonrpc_response_prepend_notice(char **response_io, const char *not * must remove any untrusted field with the same name before invoking this. */ bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, const cbm_index_resource_policy_t *policy); +bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resource_policy_t *policy, + char *error, size_t error_size); +char *cbm_mcp_index_worker_resource_response(const char *args, + const cbm_index_worker_result_t *worker_result); /* Encode a doc_candidates setting the user set to a boolean on an internal * worker request (the worker reads no config). Unset or unparseable adds no diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 92e23de64f..e14bfb7f76 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1195,7 +1195,8 @@ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p) { } const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p) { - return p && cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL; + return p && cbm_index_policy_discovery_enabled(&p->resource_policy) ? &p->resource_policy + : NULL; } cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p) { @@ -3941,7 +3942,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { .ignore_file = NULL, .max_file_size = 0, .resource_policy = - cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL, + cbm_index_policy_discovery_enabled(&p->resource_policy) ? &p->resource_policy : NULL, .resource_violation = &p->resource_violation, }; cbm_file_info_t *files = NULL; diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index a923043ca4..5fe4e618f5 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1650,6 +1650,8 @@ typedef struct { cbm_project_lock_manager_t *project_locks; bool project_lock_busy_is_error; cbm_proc_outcome_t outcomes[APP_FAKE_MAX_ATTEMPTS]; + cbm_index_resource_violation_t resource_violations[APP_FAKE_MAX_ATTEMPTS]; + bool resource_probe_failures[APP_FAKE_MAX_ATTEMPTS]; const char *responses[APP_FAKE_MAX_ATTEMPTS]; const char *marker_payloads[APP_FAKE_MAX_ATTEMPTS]; char marker_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; @@ -1754,7 +1756,9 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem static bool app_wait_for_atomic_bool(atomic_bool *value, bool expected); static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, int attempt, - const char *max_files, const char *max_source_mb) { + const char *max_files, const char *max_source_mb, + const char *max_rss_mb, + const char *max_duration_seconds) { if (!context || attempt < 0 || attempt >= APP_FAKE_MAX_ATTEMPTS) { return false; } @@ -1771,9 +1775,18 @@ static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, in yyjson_val *bytes = policy && yyjson_is_obj(policy) ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB) : NULL; - bool equal = files && bytes && yyjson_is_str(files) && yyjson_is_str(bytes) && + yyjson_val *rss = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_RSS_MB) + : NULL; + yyjson_val *duration = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS) + : NULL; + bool equal = files && bytes && rss && duration && yyjson_is_str(files) && + yyjson_is_str(bytes) && yyjson_is_str(rss) && yyjson_is_str(duration) && strcmp(yyjson_get_str(files), max_files) == 0 && - strcmp(yyjson_get_str(bytes), max_source_mb) == 0; + strcmp(yyjson_get_str(bytes), max_source_mb) == 0 && + strcmp(yyjson_get_str(rss), max_rss_mb) == 0 && + strcmp(yyjson_get_str(duration), max_duration_seconds) == 0; yyjson_doc_free(document); return equal; } @@ -1815,6 +1828,12 @@ static cbm_index_worker_poll_t app_fake_worker_poll(void *opaque, worker->result.outcome = outcome; worker->result.exit_code = outcome == CBM_PROC_CLEAN ? 0 : -1; worker->result.tree_quiesced = true; + if (worker->attempt < APP_FAKE_MAX_ATTEMPTS) { + worker->result.resource_violation = + worker->context->resource_violations[worker->attempt]; + worker->result.resource_probe_failed = + worker->context->resource_probe_failures[worker->attempt]; + } const char *response = worker->attempt < APP_FAKE_MAX_ATTEMPTS && worker->context->responses[worker->attempt] ? worker->context->responses[worker->attempt] @@ -2385,11 +2404,13 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool dirs_ok = cbm_mkdtemp(root) != NULL && cbm_mkdtemp(cache) != NULL; bool cache_set = dirs_ok && cache_saved && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0; cbm_config_t *stored_config = cache_set ? cbm_config_open(cache) : NULL; - bool config_ready = stored_config && - cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && - cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0; + bool config_ready = + stored_config && cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && + cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0; char canonical_root[APP_TEST_PATH_CAP] = {0}; bool canonical = dirs_ok && cbm_canonical_path(root, canonical_root, sizeof(canonical_root)); char *project = canonical ? cbm_project_name_from_path(canonical_root) : NULL; @@ -2434,7 +2455,8 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool first_owned = first_initialized && project && app_wait_for_subscribers(application, project, 1) && app_wait_for_atomic_int(&fake.starts, 1); - bool auto_policy_propagated = first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4"); + bool auto_policy_propagated = + first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4", "64", "7"); bool second_initialized = app_test_initialize_profile(&callbacks, sessions[1], root, CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); bool coalesced = first_owned && second_initialized && project && @@ -2777,9 +2799,11 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { ASSERT_NOT_NULL(root); ASSERT_NOT_NULL(cache); cbm_config_t *stored_config = cbm_config_open(cache); - bool configured = stored_config && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0; + bool configured = + stored_config && cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0; app_fake_worker_context_t fake; app_fake_worker_context_init(&fake); @@ -2799,7 +2823,8 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { cbm_daemon_application_t *application = configured ? cbm_daemon_application_new(&config) : NULL; int index_rc = application ? cbm_daemon_application_index(application, "policy-programmatic", root) : -1; - bool policy_propagated = index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6"); + bool policy_propagated = + index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6", "64", "7"); bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); cbm_daemon_application_free(application); @@ -2884,6 +2909,87 @@ TEST(daemon_application_programmatic_index_forwards_set_doc_candidates) { ASSERT_EQ(garbage, -1); PASS(); } + +TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { + char *cache = th_mktempdir("cbm_app_worker_resource_cache"); + cbm_config_t *stored_config = cache ? cbm_config_open(cache) : NULL; + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + atomic_store(&fake.scripted, true); + fake.outcomes[0] = CBM_PROC_KILLED; + fake.resource_violations[0] = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_DURATION_MS, .observed = 7001, .limit = 7000}; + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = { + .config = stored_config, + .worker_ops = &worker_ops, + }; + cbm_daemon_application_t *application = + stored_config ? cbm_daemon_application_new(&config) : NULL; + cbm_daemon_runtime_application_callbacks_t callbacks = + cbm_daemon_application_runtime_callbacks(application); + cbm_daemon_runtime_application_session_t *session = app_test_open(&callbacks, 4188); + char *root = th_mktempdir("cbm_app_worker_resource"); + uint8_t *context = NULL; + uint32_t context_length = 0; + uint8_t *tool = NULL; + uint32_t tool_length = 0; + char args[APP_TEST_PATH_CAP + 96]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"DaemonWorkerResourceFixture\"}", + root ? root : ""); + bool setup = cache && stored_config && application && session && root && + app_test_context_request(root, root, &context, &context_length) && + app_test_tool_request("index_repository", args, &tool, &tool_length); + uint8_t *response = NULL; + uint32_t response_length = 0; + if (setup) { + uint8_t *context_response = NULL; + uint32_t context_response_length = 0; + setup = app_test_request(&callbacks, session, context, context_length, &context_response, + &context_response_length) == CBM_DAEMON_RUNTIME_APPLICATION_OK; + free(context_response); + } + cbm_daemon_runtime_application_status_t status = + setup + ? app_test_request(&callbacks, session, tool, tool_length, &response, &response_length) + : CBM_DAEMON_RUNTIME_APPLICATION_TRANSPORT_ERROR; + bool structured = response && strstr((char *)response, "resource_limit_exceeded") && + strstr((char *)response, "\\\"stage\\\":\\\"worker\\\"") && + strstr((char *)response, "\\\"resource\\\":\\\"duration_ms\\\"") && + strstr((char *)response, "\\\"observed\\\":7001") && + strstr((char *)response, "\\\"limit\\\":7000") && + strstr((char *)response, "\\\"unit\\\":\\\"milliseconds\\\""); + if (session) { + callbacks.session_close(callbacks.context, session); + } + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + int starts = atomic_load(&fake.starts); + int destroys = atomic_load(&fake.destroys); + cbm_daemon_application_free(application); + cbm_config_close(stored_config); + free(response); + free(context); + free(tool); + th_cleanup(root); + th_cleanup(cache); + + ASSERT_TRUE(setup); + ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_OK); + ASSERT_TRUE(structured); + ASSERT_EQ(starts, 1); + ASSERT_EQ(destroys, 1); + ASSERT_TRUE(stopped); + PASS(); +} + TEST(daemon_application_auto_index_honors_tracked_file_limit) { app_env_backup_t cache_environment; bool cache_saved = app_env_backup_capture(&cache_environment, "CBM_CACHE_DIR"); @@ -6953,6 +7059,7 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_unnamed_async_and_status_refuse_ambiguous_root_issue2134); RUN_TEST(daemon_application_programmatic_index_injects_resource_policy); RUN_TEST(daemon_application_programmatic_index_forwards_set_doc_candidates); + RUN_TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried); RUN_TEST(daemon_application_auto_index_honors_tracked_file_limit); RUN_TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path); RUN_TEST(daemon_application_auto_index_file_count_supports_non_git_roots); diff --git a/tests/test_index_policy.c b/tests/test_index_policy.c index d866a06313..b33d64cdb0 100644 --- a/tests/test_index_policy.c +++ b/tests/test_index_policy.c @@ -21,9 +21,15 @@ TEST(index_policy_defaults_are_disabled) { ASSERT_FALSE(policy.max_files.enabled); ASSERT_FALSE(policy.max_source_bytes.enabled); + ASSERT_FALSE(policy.max_rss_bytes.enabled); + ASSERT_FALSE(policy.max_duration_ms.enabled); ASSERT_FALSE(cbm_index_policy_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_discovery_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_worker_enabled(&policy)); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_FILES), "off"); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_SOURCE_MB), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_RSS_MB), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_DURATION_SECONDS), "off"); PASS(); } @@ -60,6 +66,38 @@ TEST(index_policy_source_limit_converts_mib_without_overflow) { PASS(); } +TEST(index_policy_worker_limits_validate_and_convert_units) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "64", error, sizeof(error))); + ASSERT_TRUE(cbm_index_policy_enabled(&policy)); + ASSERT_TRUE(cbm_index_policy_worker_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_discovery_enabled(&policy)); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(64) * CBM_INDEX_MIB_BYTES); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "1048576", error, + sizeof(error))); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(1048576) * CBM_INDEX_MIB_BYTES); + cbm_index_resource_policy_t before = policy; + ASSERT_FALSE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "63", error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "1", error, + sizeof(error))); + ASSERT_EQ(policy.max_duration_ms.value, 1000); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "86400", error, + sizeof(error))); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(86400000)); + before = policy; + ASSERT_FALSE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "86401", + error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + PASS(); +} + TEST(index_policy_invalid_value_is_rejected_atomically) { static const char *const invalid[] = {"", "0", "-1", "1MB", "1 ", "+1", "10000001", "18446744073709551616"}; @@ -95,6 +133,16 @@ TEST(index_policy_format_round_trips_public_values) { ASSERT_TRUE( cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, value, sizeof(value))); ASSERT_STR_EQ(value, "7"); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "64", error, sizeof(error))); + ASSERT_TRUE( + cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, value, sizeof(value))); + ASSERT_STR_EQ(value, "64"); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "9", error, + sizeof(error))); + ASSERT_TRUE(cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, value, + sizeof(value))); + ASSERT_STR_EQ(value, "9"); PASS(); } @@ -103,10 +151,18 @@ TEST(index_policy_violation_metadata_is_stable) { ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_SOURCE_BYTES), "source_bytes"); ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_FILES), "files"); ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_SOURCE_BYTES), "bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_RSS_BYTES), "rss_bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_DURATION_MS), "duration_ms"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_RSS_BYTES), "bytes"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_DURATION_MS), "milliseconds"); ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_FILES), CBM_INDEX_CONFIG_MAX_FILES); ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_SOURCE_BYTES), CBM_INDEX_CONFIG_MAX_SOURCE_MB); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_RSS_BYTES), + CBM_INDEX_CONFIG_MAX_RSS_MB); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_DURATION_MS), + CBM_INDEX_CONFIG_MAX_DURATION_SECONDS); PASS(); } @@ -122,9 +178,13 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { ASSERT_FALSE(cbm_index_policy_enabled(&policy)); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "9"), 0); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "3"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7"), 0); ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); ASSERT_EQ(policy.max_files.value, 9); ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(3) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(64) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(7000)); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "corrupt"), 0); ASSERT_FALSE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); @@ -135,16 +195,23 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { PASS(); } -TEST(index_policy_cli_lists_both_operator_keys) { +TEST(index_policy_cli_lists_all_operator_keys) { bool files_found = false; bool bytes_found = false; + bool rss_found = false; + bool duration_found = false; for (size_t index = 0; index < cbm_cli_config_key_count_for_testing(); index++) { const char *key = cbm_cli_config_key_at_for_testing(index); files_found = files_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0); bytes_found = bytes_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); + rss_found = rss_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_RSS_MB) == 0); + duration_found = + duration_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS) == 0); } ASSERT_TRUE(files_found); ASSERT_TRUE(bytes_found); + ASSERT_TRUE(rss_found); + ASSERT_TRUE(duration_found); PASS(); } @@ -391,11 +458,12 @@ SUITE(index_policy) { RUN_TEST(index_policy_defaults_are_disabled); RUN_TEST(index_policy_file_limit_accepts_off_and_exact_range); RUN_TEST(index_policy_source_limit_converts_mib_without_overflow); + RUN_TEST(index_policy_worker_limits_validate_and_convert_units); RUN_TEST(index_policy_invalid_value_is_rejected_atomically); RUN_TEST(index_policy_format_round_trips_public_values); RUN_TEST(index_policy_violation_metadata_is_stable); RUN_TEST(index_policy_config_loads_defaults_values_and_rejects_corruption); - RUN_TEST(index_policy_cli_lists_both_operator_keys); + RUN_TEST(index_policy_cli_lists_all_operator_keys); RUN_TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite); RUN_TEST(index_policy_cli_set_reports_a_failed_write); RUN_TEST(index_policy_worker_rejects_missing_parent_policy); diff --git a/tests/test_index_supervisor.c b/tests/test_index_supervisor.c index 1861c8076c..88d58c0d70 100644 --- a/tests/test_index_supervisor.c +++ b/tests/test_index_supervisor.c @@ -1019,6 +1019,269 @@ TEST(index_supervisor_job_memory_limit_has_floor_headroom_and_no_overflow) { PASS(); } +typedef struct { + uint64_t now_ms; + uint64_t rss_values[4]; + int rss_value_count; + int rss_calls; + int clock_calls; + cbm_proc_tree_rss_status_t rss_status; +} index_supervisor_resource_fake_t; + +static uint64_t index_supervisor_fake_clock(void *context) { + index_supervisor_resource_fake_t *fake = context; + fake->clock_calls++; + return fake->now_ms; +} + +static cbm_proc_tree_rss_status_t index_supervisor_fake_rss(cbm_subprocess_t *process, + uint64_t *rss_bytes, void *context) { + (void)process; + index_supervisor_resource_fake_t *fake = context; + int value_index = + fake->rss_calls < fake->rss_value_count ? fake->rss_calls : fake->rss_value_count - 1; + fake->rss_calls++; + if (fake->rss_status == CBM_PROC_TREE_RSS_OK && rss_bytes && value_index >= 0) { + *rss_bytes = fake->rss_values[value_index]; + } + return fake->rss_status; +} + +static cbm_index_resource_policy_t index_supervisor_test_worker_policy(uint64_t rss_bytes, + uint64_t duration_ms) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_rss_bytes = (cbm_index_limit_u64_t){.enabled = rss_bytes > 0, .value = rss_bytes}; + policy.max_duration_ms = + (cbm_index_limit_u64_t){.enabled = duration_ms > 0, .value = duration_ms}; + return policy; +} + +TEST(index_supervisor_disabled_limits_do_not_probe) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {UINT64_MAX}, + .rss_value_count = 1, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"clean\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + cbm_index_resource_t resource = + result ? result->resource_violation.resource : CBM_INDEX_RESOURCE_RSS_BYTES; + bool probe_failed = result && result->resource_probe_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(terminal); + ASSERT_EQ(fake.rss_calls, 0); + ASSERT_EQ(fake.clock_calls, 0); + ASSERT_EQ(resource, CBM_INDEX_RESOURCE_NONE); + ASSERT_FALSE(probe_failed); + PASS(); +} + +TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {limit, limit + 1}, + .rss_value_count = 2, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t equal_state = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms += 250; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + const cbm_index_worker_result_t *cached = NULL; + bool cached_terminal = + terminal && cbm_index_worker_poll(handle, &cached) == CBM_INDEX_WORKER_POLL_TERMINAL && + cached == result; + bool limited = terminal && result && + result->resource_violation.resource == CBM_INDEX_RESOURCE_RSS_BYTES && + result->resource_violation.observed == limit + 1 && + result->resource_violation.limit == limit && !result->cancellation_requested && + result->tree_quiesced && !result->supervision_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(equal_state, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(limited); + ASSERT_TRUE(cached_terminal); + PASS(); +} + +TEST(index_supervisor_duration_is_total_time_not_quiet_timeout) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_EMPTY, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + fake.now_ms = 1100; + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t equal_state = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms = 1101; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool limited = terminal && result && + result->resource_violation.resource == CBM_INDEX_RESOURCE_DURATION_MS && + result->resource_violation.observed == 1001 && + result->resource_violation.limit == 1000 && result->outcome != CBM_PROC_HANG && + result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(equal_state, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(limited); + PASS(); +} + +TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled) { + const char *saved_timeout = getenv("CBM_INDEX_WORKER_TIMEOUT_S"); + char *saved_timeout_copy = saved_timeout ? cbm_strdup(saved_timeout) : NULL; + (void)cbm_setenv("CBM_INDEX_WORKER_TIMEOUT_S", "1", 1); + cbm_index_supervisor_reset_resource_hooks_for_testing(); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 60000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool remained_hang = terminal && result && result->outcome == CBM_PROC_HANG && + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + index_supervisor_test_restore_env("CBM_INDEX_WORKER_TIMEOUT_S", saved_timeout_copy); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(terminal); + ASSERT_TRUE(remained_hang); + PASS(); +} + +TEST(index_supervisor_cancel_precedes_resource_probe) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {limit + 1}, + .rss_value_count = 1, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + fake.now_ms = 2000; + bool cancel_accepted = handle && cbm_index_worker_request_cancel(handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool cancelled = terminal && result && result->cancellation_requested && + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed && result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(cancelled); + ASSERT_EQ(fake.rss_calls, 0); + PASS(); +} + +TEST(index_supervisor_three_failed_rss_probes_fail_closed) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_ERROR, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t first = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + cbm_index_worker_poll_t throttled = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + int calls_before_interval = fake.rss_calls; + fake.now_ms += 250; + cbm_index_worker_poll_t second = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms += 250; + cbm_index_worker_poll_t third = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool failed_closed = terminal && result && result->resource_probe_failed && + result->resource_violation.resource == CBM_INDEX_RESOURCE_RSS_BYTES && + !result->cancellation_requested && result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(first, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(throttled, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(calls_before_interval, 1); + ASSERT_EQ(second, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(third, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(failed_closed); + ASSERT_EQ(fake.rss_calls, 3); + PASS(); +} + SUITE(index_supervisor) { RUN_TEST(index_supervisor_job_memory_limit_has_floor_headroom_and_no_overflow); RUN_TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar); @@ -1030,4 +1293,10 @@ SUITE(index_supervisor) { RUN_TEST(index_supervisor_oversized_response_is_contained_and_log_is_retained); RUN_TEST(index_supervisor_clean_exit_without_response_is_named_failure_issue1300); RUN_TEST(index_supervisor_killed_worker_log_is_never_empty_and_names_the_run); + RUN_TEST(index_supervisor_disabled_limits_do_not_probe); + RUN_TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree); + RUN_TEST(index_supervisor_duration_is_total_time_not_quiet_timeout); + RUN_TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled); + RUN_TEST(index_supervisor_cancel_precedes_resource_probe); + RUN_TEST(index_supervisor_three_failed_rss_probes_fail_closed); } diff --git a/tests/test_mcp.c b/tests/test_mcp.c index ebf24a827f..3bfcb018ff 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -21581,6 +21581,103 @@ TEST(index_supervisor_clean_exit_without_response_response_issue1300) { PASS(); } +#ifndef _WIN32 +typedef struct { + int calls; +} index_worker_resource_clock_t; + +static uint64_t index_worker_resource_clock(void *context) { + index_worker_resource_clock_t *clock = context; + clock->calls++; + return clock->calls == 1 ? 100 : 1101; +} + +enum { + IDXRESOURCE_OK = 0, + IDXRESOURCE_SETUP = 71, + IDXRESOURCE_NO_RESPONSE = 72, + IDXRESOURCE_BAD_CONTRACT = 73, + IDXRESOURCE_RETRIED = 74, +}; + +static int index_worker_resource_response_check(const char *repo, const char *cache) { + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + cbm_unsetenv("CBM_INDEX_SUPERVISOR"); + cbm_index_supervisor_mark_host(); + cbm_config_t *config = cbm_config_open(cache); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + if (!config || !server || + cbm_config_set(config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "1") != 0) { + cbm_mcp_server_free(server); + cbm_config_close(config); + return IDXRESOURCE_SETUP; + } + cbm_mcp_server_set_config(server, config); + index_worker_resource_clock_t clock = {0}; + cbm_index_supervisor_set_resource_hooks_for_testing(index_worker_resource_clock, NULL, &clock); + int before = cbm_index_supervisor_spawn_count(); + char args[CBM_SZ_2K]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"mode\":\"fast\"," + "\"_cbm_index_policy\":{\"index_max_files\":\"off\"," + "\"index_max_source_mb\":\"off\",\"index_max_rss_mb\":\"off\"," + "\"index_max_duration_seconds\":\"off\"}}", + repo); + char *response = cbm_mcp_handle_tool(server, "index_repository", args); + int after = cbm_index_supervisor_spawn_count(); + cbm_index_supervisor_reset_resource_hooks_for_testing(); + int result = IDXRESOURCE_OK; + if (!response) { + result = IDXRESOURCE_NO_RESPONSE; + } else if (!strstr(response, "resource_limit_exceeded") || + !strstr(response, "\\\"stage\\\":\\\"worker\\\"") || + !strstr(response, "\\\"resource\\\":\\\"duration_ms\\\"") || + !strstr(response, "\\\"observed\\\":1001") || + !strstr(response, "\\\"limit\\\":1000") || + !strstr(response, "\\\"unit\\\":\\\"milliseconds\\\"")) { + result = IDXRESOURCE_BAD_CONTRACT; + } else if (after - before != 1) { + result = IDXRESOURCE_RETRIED; + } + free(response); + cbm_mcp_server_free(server); + cbm_config_close(config); + return result; +} +#endif + +TEST(index_worker_resource_limit_is_trusted_structured_and_not_retried) { +#ifdef _WIN32 + SKIP_PLATFORM("fork-isolated MCP host harness; supervisor state machine is cross-platform"); +#else + char *repo = th_mktempdir("cbm_worker_resource_repo"); + char *cache = th_mktempdir("cbm_worker_resource_cache"); + ASSERT_NOT_NULL(repo); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 0; }\n"), 0); + fflush(NULL); + pid_t pid = fork(); + if (pid == 0) { + alarm(60); + _exit(index_worker_resource_response_check(repo, cache)); + } + int status = 0; + bool waited = waitpid(pid, &status, 0) == pid; + int exit_code = waited && WIFEXITED(status) ? WEXITSTATUS(status) : -1; + int signal_code = waited && WIFSIGNALED(status) ? WTERMSIG(status) : 0; + th_cleanup(repo); + th_cleanup(cache); + if (exit_code != IDXRESOURCE_OK) { + printf(" worker resource child exit=%d signal=%d\n", exit_code, signal_code); + } + ASSERT_TRUE(waited); + ASSERT_EQ(signal_code, 0); + ASSERT_EQ(exit_code, IDXRESOURCE_OK); + PASS(); +#endif +} + + /* Child-side check: index a tiny fixture and verify it ran IN-PROCESS. * Distinct exit codes so the parent can report the exact failure mode. */ enum { @@ -25140,6 +25237,7 @@ SUITE(mcp) { RUN_TEST(index_supervisor_gate_requires_marked_host_issue845); RUN_TEST(index_supervisor_start_failure_is_fail_closed_in_real_host); RUN_TEST(index_supervisor_clean_exit_without_response_response_issue1300); + RUN_TEST(index_worker_resource_limit_is_trusted_structured_and_not_retried); RUN_TEST(index_bg_paths_route_through_supervisor_issue832); RUN_TEST(sequential_service_edge_props_are_valid_json_issue898); RUN_TEST(index_second_inprocess_run_survives_issue773); diff --git a/tests/test_subprocess.c b/tests/test_subprocess.c index c7f78b3d87..385293e085 100644 --- a/tests/test_subprocess.c +++ b/tests/test_subprocess.c @@ -101,6 +101,12 @@ TEST(subprocess_outcome_str) { PASS(); } +TEST(subprocess_tree_rss_sum_saturates_on_overflow) { + const uint64_t values[] = {UINT64_MAX - 4, 4, 1}; + ASSERT_EQ(cbm_subprocess_rss_sum_for_testing(values, 3), UINT64_MAX); + PASS(); +} + /* ── Layer 2: real spawn/reap (POSIX) ─────────────────────────────────────── */ extern const char *cbm_test_program_path; @@ -801,6 +807,45 @@ TEST(subprocess_spawn_returns_while_child_is_running) { #endif } +TEST(subprocess_tree_rss_measures_contained_descendants) { +#ifdef _WIN32 + SKIP_PLATFORM("native Job Object process-tree RSS runs on Windows CI"); +#else + char pid_path[64]; + ASSERT_TRUE(make_tree_pid_path(pid_path)); + cbm_subprocess_t *process = NULL; + ASSERT_EQ(spawn_ignoring_tree(pid_path, 0, 100, &process), 0); + pid_t parent_pid = 0; + pid_t grandchild_pid = 0; + bool ready = wait_for_tree_pids(pid_path, process, &parent_pid, &grandchild_pid, 3000); + uint64_t rss_bytes = 0; + cbm_proc_tree_rss_status_t rss_status = + ready ? cbm_subprocess_tree_rss_bytes(process, &rss_bytes) : CBM_PROC_TREE_RSS_ERROR; + bool cancel_accepted = cbm_subprocess_request_cancel(process); + cbm_proc_result_t result; + bool terminal = poll_until_terminal(process, 5000, &result); + uint64_t final_rss = UINT64_MAX; + cbm_proc_tree_rss_status_t final_rss_status = + terminal ? cbm_subprocess_tree_rss_bytes(process, &final_rss) : CBM_PROC_TREE_RSS_ERROR; + if (terminal) { + cbm_subprocess_destroy(process); + } else { + force_probe_cleanup(parent_pid, grandchild_pid); + } + (void)unlink(pid_path); + + ASSERT_TRUE(ready); + ASSERT_EQ(rss_status, CBM_PROC_TREE_RSS_OK); + ASSERT_TRUE(rss_bytes > 0); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(terminal); + ASSERT_TRUE(result.tree_quiesced); + ASSERT_EQ(final_rss_status, CBM_PROC_TREE_RSS_EMPTY); + ASSERT_EQ(final_rss, 0); + PASS(); +#endif +} + TEST(subprocess_natural_completion_is_cached_across_polls) { #ifdef _WIN32 SKIP_PLATFORM("POSIX /bin/sh completion-cache probe; native Job Object coverage pending"); @@ -2635,6 +2680,7 @@ SUITE(subprocess) { #ifndef _WIN32 RUN_TEST(subprocess_environment_path_preserves_caller_lookup); #endif + RUN_TEST(subprocess_tree_rss_sum_saturates_on_overflow); RUN_TEST(subprocess_run_clean); RUN_TEST(subprocess_run_exit_nonzero); RUN_TEST(subprocess_run_child_exited_before_group_check); @@ -2648,6 +2694,7 @@ SUITE(subprocess) { RUN_TEST(subprocess_run_spawn_failure); RUN_TEST(subprocess_run_null_bin_rejected); RUN_TEST(subprocess_spawn_returns_while_child_is_running); + RUN_TEST(subprocess_tree_rss_measures_contained_descendants); RUN_TEST(subprocess_natural_completion_is_cached_across_polls); RUN_TEST(subprocess_cancel_is_idempotent_and_kills_ignoring_tree); RUN_TEST(subprocess_quiet_timeout_kills_ignoring_tree); From a45858ffb5a53a3c4d7707d7a237f7f37d749aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Tue, 22 Sep 2026 01:28:50 +0800 Subject: [PATCH 08/27] fix(test): send the complete worker policy from the session-scope fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR 2257 still named only the two discovery keys in the worker-scope script. The worker now fail-closes unless every known policy key is present, so the fixture must stand in for the supervisor with cbm_test_index_worker_policy_json. Signed-off-by: 刘冲 --- tests/test_worker_session_scope.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_worker_session_scope.sh b/tests/test_worker_session_scope.sh index ab018ba765..1f32c46827 100644 --- a/tests/test_worker_session_scope.sh +++ b/tests/test_worker_session_scope.sh @@ -73,9 +73,9 @@ run_worker() { # A worker never reads the resource policy from config or environment: the # supervisor resolves it and sends it inside the request, and a request # without one is refused ("missing or incomplete trusted worker policy"). -# This script plays the supervisor, so it sends what the supervisor sends -- -# both limits off, the default. -policy='"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"}' +# This script plays the supervisor, so it sends the same complete object +# cbm_test_index_worker_policy_json produces -- every known key, all off. +policy="$(cbm_test_index_worker_policy_json)" response="${tmpdir}/scoped.response" if ! run_worker "{\"repo_path\":\"${repo}\",\"mode\":\"fast\",${policy}}" "${response}" \ From 1e4751d3ca4dab07fef38416729ff7b445d8cedf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Tue, 22 Sep 2026 03:05:13 +0800 Subject: [PATCH 09/27] fix(index): charge macOS phys_footprint and pin duration across recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Watchdogs now sample the OS-charged footprint instead of resident size on Darwin, and index_max_duration_seconds covers the whole request so a crash/hang respawn cannot reset the clock. Signed-off-by: 刘冲 --- README.md | 4 +- docs/CONFIGURATION.md | 4 +- docs/INDEX_RESOURCE_LIMITS.md | 28 ++++++---- src/cli/cli.c | 5 +- src/daemon/application.c | 13 +++-- src/daemon/application.h | 2 +- src/foundation/subprocess.c | 12 ++++- src/foundation/subprocess.h | 10 ++-- src/mcp/index_supervisor.c | 36 ++++++++----- src/mcp/index_supervisor.h | 12 +++-- src/mcp/mcp.c | 8 +-- tests/test_daemon_application.c | 2 + tests/test_index_supervisor.c | 94 ++++++++++++++++++++++++++------- 13 files changed, 166 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 4466dae50b..c34d20b982 100644 --- a/README.md +++ b/README.md @@ -928,8 +928,8 @@ codebase-memory-mcp config set watcher_enabled false # stop the watcher thre codebase-memory-mcp config set watch_non_git true # also poll non-git project roots (default: false) codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit -codebase-memory-mcp config set index_max_rss_mb 8192 # optional worker-tree current RSS limit -codebase-memory-mcp config set index_max_duration_seconds 3600 # optional total worker duration +codebase-memory-mcp config set index_max_rss_mb 8192 # optional worker-tree charged-memory limit +codebase-memory-mcp config set index_max_duration_seconds 3600 # optional per-request duration codebase-memory-mcp config reset auto_index # reset to default ``` diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 3d3e9a9716..a02459b4ab 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -93,8 +93,8 @@ Current keys: | `watch_non_git` | `false` | Also poll project roots that are **not git repositories**. By default the watcher only follows git projects, so a project indexed from a plain directory is never refreshed after its first index — reindex it manually with `index_repository`. Set `true` to poll such roots on the same adaptive cadence with a file-tree scan: the indexer's own discovery walk (same skip lists, `.gitignore` and `.cbmignore` rules) hashed over each file's path, size and mtime. Any change reindexes once; paths the indexer skips (including cbm's own `.codebase-memory/` output) never trigger. The first poll after the daemon starts reindexes each such project once, since nothing records which tree state the index holds. The scan walks the whole tree every poll, so it costs more than git polling on very large trees. Read once when the daemon starts, like `watcher_enabled`. | | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | -| `index_max_rss_mb` | `off` | Optional maximum current RSS in MiB for the complete contained index-worker process tree (`64..1048576`). | -| `index_max_duration_seconds` | `off` | Optional maximum total worker duration in seconds (`1..86400`). | +| `index_max_rss_mb` | `off` | Optional maximum charged memory in MiB for the complete contained index-worker process tree (`64..1048576`). On macOS this is phys_footprint, not resident size. | +| `index_max_duration_seconds` | `off` | Optional maximum wall-clock duration in seconds for the whole index request, including crash/hang recovery (`1..86400`). | For a watched project, `index_status` names the polling method in `watch.strategy`: `git` (HEAD and dirty-state polling), `tree` (a non-git root polled by the diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index 66be4ecf46..1525993606 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -27,24 +27,30 @@ ranges are rejected without changing the stored value. | Key | Default | Accepted value | Protects | |---|---:|---:|---| -| `index_max_rss_mb` | `off` | `off` or `64..1048576` | Current RSS of the complete worker process tree | -| `index_max_duration_seconds` | `off` | `off` or `1..86400` | Total worker wall-clock duration | +| `index_max_rss_mb` | `off` | `off` or `64..1048576` | Charged memory of the complete worker process tree | +| `index_max_duration_seconds` | `off` | `off` or `1..86400` | Wall-clock duration of the whole index request | ```bash codebase-memory-mcp config set index_max_rss_mb 8192 codebase-memory-mcp config set index_max_duration_seconds 3600 ``` -RSS is the current resident memory of the contained worker and every descendant, -not the worker's allocation budget and not peak memory. This hard watchdog is +RSS is the charged memory of the contained worker and every descendant, not +the worker's allocation budget and not peak memory. On macOS that quantity is +`phys_footprint` (the same number `cbm_mem_charged()` enforces), not +`resident_size`, which still counts pages the allocator has already handed +back. Linux and Windows use RSS / working set. This hard watchdog is separate from the internal `CBM_MEM_BUDGET_MB` soft budget. The supervisor -samples RSS at most once every 250 milliseconds so the watchdog does not turn -full process-table enumeration into a busy loop. - -Duration uses a monotonic clock from successful spawn. It is independent of the -existing 15-minute quiet timeout: continuous log progress does not reset total -duration, while the quiet timeout continues to identify a worker that stops -making progress. +samples the tree at most once every 250 milliseconds so the watchdog does not +turn full process-table enumeration into a busy loop. + +Duration is per request: the clock starts at the first worker spawn and is +not reset when crash/hang recovery starts a later attempt. Continuous log +progress does not reset it. It is independent of the existing 15-minute quiet +timeout, which still identifies a worker that stops making progress. A +duration limit shorter than that quiet timeout kills a hung worker before hang +quarantine can name the in-flight file, so the next attempt may hang on the +same file. Equality is allowed. The first RSS or elapsed-duration observation above its limit starts the existing graceful-to-force process-tree shutdown. CBM reports diff --git a/src/cli/cli.c b/src/cli/cli.c index 00b2db3410..fd51c48e9f 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -7603,8 +7603,9 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"}, {CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"}, {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"}, - {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree RSS MiB, or off"}, - {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "off", "Max worker duration in seconds, or off"}, + {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree charged memory MiB, or off"}, + {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "off", + "Max wall-clock seconds for the whole index request, or off"}, }; /* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing diff --git a/src/daemon/application.c b/src/daemon/application.c index a7b2f4f5b0..0f696d59c4 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -159,6 +159,7 @@ struct cbm_daemon_application_job { * job outlives the request and session that started it. */ bool async_owned; bool worker_started; + uint64_t request_started_ms; cbm_daemon_application_job_t *next; }; @@ -361,6 +362,7 @@ static void application_worker_diagnostic(const char *line, void *context) { static int application_worker_start_default(void *context, const char *args_json, size_t memory_budget_bytes, const char *marker_file, const char *quarantine_file, + uint64_t duration_origin_ms, cbm_daemon_application_worker_t *worker_out) { (void)context; cbm_index_resource_policy_t resource_policy; @@ -382,7 +384,7 @@ static int application_worker_start_default(void *context, const char *args_json yyjson_doc_free(args_doc); int result = cbm_index_worker_start_with_policy_log( args_json, memory_budget_bytes, &resource_policy, false, marker_file, quarantine_file, - application_worker_diagnostic, worker, &worker->handle); + application_worker_diagnostic, worker, duration_origin_ms, &worker->handle); if (result != 0) { application_worker_destroy_default(NULL, worker); *worker_out = NULL; @@ -1377,10 +1379,13 @@ static application_attempt_status_t application_job_run_attempt(cbm_daemon_appli } cbm_daemon_application_worker_t worker = NULL; + if (job->request_started_ms == 0) { + job->request_started_ms = cbm_index_worker_now_ms(); + } application_tmp_lock(); - int start_result = - application->worker_ops.start(application->worker_ops.context, job->args_json, - memory_budget_bytes, marker_path, quarantine_path, &worker); + int start_result = application->worker_ops.start( + application->worker_ops.context, job->args_json, memory_budget_bytes, marker_path, + quarantine_path, job->request_started_ms, &worker); application_tmp_unlock(); if (start_result != 0 || !worker) { return application_job_cancel_requested(job) ? APPLICATION_ATTEMPT_CANCELLED diff --git a/src/daemon/application.h b/src/daemon/application.h index 933186aa73..426bef3db9 100644 --- a/src/daemon/application.h +++ b/src/daemon/application.h @@ -31,7 +31,7 @@ typedef void *cbm_daemon_application_update_worker_t; typedef struct { void *context; int (*start)(void *context, const char *args_json, size_t memory_budget_bytes, - const char *marker_file, const char *quarantine_file, + const char *marker_file, const char *quarantine_file, uint64_t duration_origin_ms, cbm_daemon_application_worker_t *worker_out); cbm_index_worker_poll_t (*poll)(void *context, cbm_daemon_application_worker_t worker, const cbm_index_worker_result_t **result_out); diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 474ada1d2b..519a554e23 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -513,6 +513,12 @@ static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subproces bool root_failed = false; for (DWORD index = 0; index < processes->NumberOfProcessIdsInList; index++) { DWORD pid = (DWORD)processes->ProcessIdList[index]; + /* GetProcessMemoryInfo documents PROCESS_QUERY_INFORMATION | + * PROCESS_VM_READ. PROCESS_QUERY_LIMITED_INFORMATION is a lesser + * alternative on newer Windows; the Job Object membership list + * already closes most of the pid-reuse window between listing and + * opening. A member we cannot open is a probe failure when it is + * the root. */ HANDLE member = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid); if (!member) { root_failed = root_failed || pid == process->process_id; @@ -592,7 +598,11 @@ static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subproces root_failed = root_failed || pids[index] == process->pid; continue; } - cbm_rss_add_saturated(&total, usage.ri_resident_size); + /* phys_footprint is what the OS charges, matching cbm_mem_charged(). + * ri_resident_size still counts pages mimalloc already marked + * MADV_FREE_REUSABLE, so a Mac worker can look like 17 GB RSS while + * the kernel holds it to 5.5 GB. */ + cbm_rss_add_saturated(&total, usage.ri_phys_footprint); measured++; } cbm_free(CBM_MEM_CLASS_OTHER, pids); diff --git a/src/foundation/subprocess.h b/src/foundation/subprocess.h index bfcb2311b7..94a8a2ed01 100644 --- a/src/foundation/subprocess.h +++ b/src/foundation/subprocess.h @@ -155,10 +155,12 @@ int cbm_subprocess_spawn(const cbm_proc_opts_t *opts, cbm_subprocess_t **out); * tree when cancel_grace_ms elapses. Callers must keep polling to make progress. */ cbm_proc_poll_t cbm_subprocess_poll(cbm_subprocess_t *process, cbm_proc_result_t *out); -/* Read the current resident-set size of the complete contained process tree. - * OK returns an overflow-safe byte total, EMPTY means the owned tree currently - * has no observable members, and ERROR means no trustworthy measurement could - * be obtained. Individual processes that exit during enumeration are ignored. */ +/* Read the current charged memory of the complete contained process tree. + * On macOS this is phys_footprint (what the OS holds against the process); + * elsewhere it is RSS. OK returns an overflow-safe byte total, EMPTY means + * the owned tree currently has no observable members, and ERROR means no + * trustworthy measurement could be obtained. Individual processes that exit + * during enumeration are ignored. */ cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_bytes(cbm_subprocess_t *process, uint64_t *rss_bytes); bool cbm_subprocess_root_running(const cbm_subprocess_t *process); diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index 4ffeecbb89..e84dff7557 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -881,7 +881,8 @@ static int worker_start_internal(const char *args_json, size_t memory_budget_byt const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, cbm_index_worker_handle_t **handle_out) { + void *log_context, uint64_t duration_origin_ms, + cbm_index_worker_handle_t **handle_out) { if (handle_out) { *handle_out = NULL; } @@ -985,34 +986,42 @@ static int worker_start_internal(const char *args_json, size_t memory_budget_byt return -1; } if (handle->resource_policy.max_duration_ms.enabled) { - handle->started_ms = worker_resource_now_ms(); + /* Duration is per request, not per spawn: a crash/hang recovery must + * not reset the clock. Callers pass the request origin; 0 means now. */ + uint64_t origin = duration_origin_ms != 0 ? duration_origin_ms : worker_resource_now_ms(); + handle->started_ms = origin; } *handle_out = handle; return 0; } +uint64_t cbm_index_worker_now_ms(void) { + return worker_resource_now_ms(); +} + int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, cbm_index_worker_handle_t **handle_out) { return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, - quarantine_file, log_callback, log_context, handle_out); + quarantine_file, log_callback, log_context, 0, handle_out); } int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out) { return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, - quarantine_file, NULL, NULL, handle_out); + quarantine_file, NULL, NULL, 0, handle_out); } int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, - const char *quarantine_file, + const char *quarantine_file, uint64_t duration_origin_ms, cbm_index_worker_handle_t **handle_out) { return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, - marker_file, quarantine_file, NULL, NULL, handle_out); + marker_file, quarantine_file, NULL, NULL, duration_origin_ms, + handle_out); } int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_budget_bytes, @@ -1020,10 +1029,11 @@ int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_ bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, + uint64_t duration_origin_ms, cbm_index_worker_handle_t **handle_out) { return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, marker_file, quarantine_file, log_callback, log_context, - handle_out); + duration_origin_ms, handle_out); } cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, @@ -1167,14 +1177,15 @@ static int worker_spawn_internal(const char *args_json, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, const atomic_int *cancel_requested, - cbm_index_worker_result_t *result) { + uint64_t duration_origin_ms, cbm_index_worker_result_t *result) { if (!result) { return -1; } worker_result_init(result); cbm_index_worker_handle_t *handle = NULL; if (worker_start_internal(args_json, 0, resource_policy, single_thread, marker_file, - quarantine_file, log_callback, log_context, &handle) != 0) { + quarantine_file, log_callback, log_context, duration_origin_ms, + &handle) != 0) { return -1; } const cbm_index_worker_result_t *cached = NULL; @@ -1206,16 +1217,17 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { return worker_spawn_internal(args_json, NULL, single_thread, marker_file, quarantine_file, - log_callback, log_context, cancel_requested, result); + log_callback, log_context, cancel_requested, 0, result); } int cbm_index_spawn_worker_with_policy_log_cancel( const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { + void *log_context, const atomic_int *cancel_requested, uint64_t duration_origin_ms, + cbm_index_worker_result_t *result) { return worker_spawn_internal(args_json, resource_policy, single_thread, marker_file, quarantine_file, log_callback, log_context, cancel_requested, - result); + duration_origin_ms, result); } int cbm_index_spawn_worker_with_log(const char *args_json, bool single_thread, diff --git a/src/mcp/index_supervisor.h b/src/mcp/index_supervisor.h index a86761d773..2cb244a7ee 100644 --- a/src/mcp/index_supervisor.h +++ b/src/mcp/index_supervisor.h @@ -186,19 +186,24 @@ int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bo int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, - const char *quarantine_file, + const char *quarantine_file, uint64_t duration_origin_ms, cbm_index_worker_handle_t **handle_out); /* Policy plus the request-scoped log callback. The daemon default start path * needs both: a missing policy fails closed, and worker diagnostics still - * reach the daemon journal. */ + * reach the daemon journal. duration_origin_ms is the request clock; 0 means now. */ int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_budget_bytes, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, + uint64_t duration_origin_ms, cbm_index_worker_handle_t **handle_out); +/* Supervisor clock used to stamp a request-scoped duration origin. Tests may + * replace it; production is cbm_now_ms(). */ +uint64_t cbm_index_worker_now_ms(void); + /* Request-scoped variant used by interactive local CLI calls. The callback is * invoked by the owner thread while it polls the contained worker; log_context * remains caller-owned until terminal. No process-global sink is installed. @@ -264,7 +269,8 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th int cbm_index_spawn_worker_with_policy_log_cancel( const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result); + void *log_context, const atomic_int *cancel_requested, uint64_t duration_origin_ms, + cbm_index_worker_result_t *result); void cbm_index_worker_result_free(cbm_index_worker_result_t *result); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 1016f781ab..f23a00e5cd 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -11617,12 +11617,14 @@ char *cbm_mcp_index_worker_no_response_failure(const char *args, static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, const cbm_index_resource_policy_t *resource_policy) { invalidate_cached_store(srv); + uint64_t request_started_ms = cbm_index_worker_now_ms(); /* First attempt: normal parallel run. */ cbm_index_worker_result_t wr; int rc = cbm_index_spawn_worker_with_policy_log_cancel( args, resource_policy, false, NULL, NULL, srv ? srv->index_log_callback : NULL, - srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wr); + srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, + request_started_ms, &wr); cbm_mcp_supervised_result_disposition_t disposition = cbm_mcp_supervised_result_disposition(rc, &wr); @@ -11704,7 +11706,7 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, int rc2 = cbm_index_spawn_worker_with_policy_log_cancel( args, resource_policy, /*single_thread=*/false, marker_path, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, - srv ? &srv->pipeline_cancel_requested : NULL, &wr2); + srv ? &srv->pipeline_cancel_requested : NULL, request_started_ms, &wr2); cbm_mcp_supervised_result_disposition_t recovery_disposition = cbm_mcp_supervised_result_disposition(rc2, &wr2); if (wr2.response_missing) { @@ -11831,7 +11833,7 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, int rcp = cbm_index_spawn_worker_with_policy_log_cancel( args, resource_policy, /*single_thread=*/false, NULL, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, - srv ? &srv->pipeline_cancel_requested : NULL, &wrp); + srv ? &srv->pipeline_cancel_requested : NULL, request_started_ms, &wrp); cbm_mcp_supervised_result_disposition_t partial_disposition = cbm_mcp_supervised_result_disposition(rcp, &wrp); if (partial_disposition == CBM_MCP_SUPERVISED_RESULT_SUCCESS) { diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index 5fe4e618f5..5641ecbac5 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1703,7 +1703,9 @@ static void app_fake_worker_read_file(const char *path, char *out, size_t out_si static int app_fake_worker_start(void *opaque, const char *args_json, size_t memory_budget_bytes, const char *marker_file, const char *quarantine_file, + uint64_t duration_origin_ms, cbm_daemon_application_worker_t *worker_out) { + (void)duration_origin_ms; app_fake_worker_context_t *context = opaque; app_fake_worker_t *worker = calloc(1, sizeof(*worker)); if (!worker) { diff --git a/tests/test_index_supervisor.c b/tests/test_index_supervisor.c index 88d58c0d70..df55407da5 100644 --- a/tests/test_index_supervisor.c +++ b/tests/test_index_supervisor.c @@ -689,8 +689,8 @@ TEST(index_supervisor_worker_keeps_default_info_liveness_heartbeat) { (void)cbm_unsetenv("CBM_LOG_LEVEL"); cbm_index_worker_handle_t *handle = NULL; - int start_rc = cbm_index_worker_start("{\"__cbm_test_worker\":\"heartbeat\"}", 0, false, - NULL, NULL, &handle); + int start_rc = cbm_index_worker_start("{\"__cbm_test_worker\":\"heartbeat\"}", 0, false, NULL, + NULL, &handle); char log_path[INDEX_SUPERVISOR_TEST_PATH_CAP] = {0}; if (handle) { (void)snprintf(log_path, sizeof(log_path), "%s", cbm_index_worker_log_path(handle)); @@ -698,8 +698,8 @@ TEST(index_supervisor_worker_keeps_default_info_liveness_heartbeat) { bool ready = log_path[0] && index_supervisor_test_wait_file_text( log_path, "async worker heartbeat probe ready", INDEX_SUPERVISOR_TEST_READY_MS); - bool heartbeat = ready && index_supervisor_test_wait_file_text( - log_path, "msg=pipeline.discover", 1000); + bool heartbeat = + ready && index_supervisor_test_wait_file_text(log_path, "msg=pipeline.discover", 1000); const cbm_index_worker_result_t *result = NULL; bool terminal = handle && index_supervisor_test_poll_terminal( handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); @@ -1034,8 +1034,9 @@ static uint64_t index_supervisor_fake_clock(void *context) { return fake->now_ms; } -static cbm_proc_tree_rss_status_t index_supervisor_fake_rss(cbm_subprocess_t *process, - uint64_t *rss_bytes, void *context) { +static cbm_proc_tree_rss_status_t index_supervisor_fake_charged_rss(cbm_subprocess_t *process, + uint64_t *rss_bytes, + void *context) { (void)process; index_supervisor_resource_fake_t *fake = context; int value_index = @@ -1065,11 +1066,11 @@ TEST(index_supervisor_disabled_limits_do_not_probe) { .rss_status = CBM_PROC_TREE_RSS_OK, }; cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, - index_supervisor_fake_rss, &fake); + index_supervisor_fake_charged_rss, &fake); cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 0); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"clean\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); const cbm_index_worker_result_t *result = NULL; bool terminal = handle && index_supervisor_test_poll_terminal( handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); @@ -1092,7 +1093,7 @@ TEST(index_supervisor_disabled_limits_do_not_probe) { PASS(); } -TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree) { +TEST(index_supervisor_charged_rss_equality_runs_then_excess_terminates_tree) { const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; index_supervisor_resource_fake_t fake = { .now_ms = 100, @@ -1101,11 +1102,11 @@ TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree) { .rss_status = CBM_PROC_TREE_RSS_OK, }; cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, - index_supervisor_fake_rss, &fake); + index_supervisor_fake_charged_rss, &fake); cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); const cbm_index_worker_result_t *result = NULL; cbm_index_worker_poll_t equal_state = handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; @@ -1141,11 +1142,11 @@ TEST(index_supervisor_duration_is_total_time_not_quiet_timeout) { .rss_status = CBM_PROC_TREE_RSS_EMPTY, }; cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, - index_supervisor_fake_rss, &fake); + index_supervisor_fake_charged_rss, &fake); cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); fake.now_ms = 1100; const cbm_index_worker_result_t *result = NULL; cbm_index_worker_poll_t equal_state = @@ -1171,6 +1172,60 @@ TEST(index_supervisor_duration_is_total_time_not_quiet_timeout) { PASS(); } +TEST(index_supervisor_duration_spans_recovery_attempts) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_EMPTY, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_charged_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); + cbm_index_worker_handle_t *first = NULL; + int first_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, 100, &first); + fake.now_ms = 500; + const cbm_index_worker_result_t *first_result = NULL; + cbm_index_worker_poll_t mid = + first ? cbm_index_worker_poll(first, &first_result) : CBM_INDEX_WORKER_POLL_ERROR; + bool cancel_accepted = first && cbm_index_worker_request_cancel(first); + bool first_terminal = first && index_supervisor_test_poll_terminal( + first, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &first_result); + if (first_terminal) { + cbm_index_worker_destroy(first); + first = NULL; + } else { + index_supervisor_test_cleanup_handle(first); + first = NULL; + } + + fake.now_ms = 1101; + cbm_index_worker_handle_t *second = NULL; + int second_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, 100, &second); + const cbm_index_worker_result_t *second_result = NULL; + bool second_terminal = second && index_supervisor_test_poll_terminal( + second, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &second_result); + bool limited = second_terminal && second_result && + second_result->resource_violation.resource == CBM_INDEX_RESOURCE_DURATION_MS && + second_result->resource_violation.observed == 1001 && + second_result->resource_violation.limit == 1000 && + second_result->outcome != CBM_PROC_HANG && second_result->tree_quiesced; + if (second_terminal) { + cbm_index_worker_destroy(second); + } else { + index_supervisor_test_cleanup_handle(second); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(first_rc, 0); + ASSERT_EQ(mid, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(first_terminal); + ASSERT_EQ(second_rc, 0); + ASSERT_TRUE(limited); + PASS(); +} + TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled) { const char *saved_timeout = getenv("CBM_INDEX_WORKER_TIMEOUT_S"); char *saved_timeout_copy = saved_timeout ? cbm_strdup(saved_timeout) : NULL; @@ -1179,7 +1234,7 @@ TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled) { cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 60000); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); const cbm_index_worker_result_t *result = NULL; bool terminal = handle && index_supervisor_test_poll_terminal( handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); @@ -1208,11 +1263,11 @@ TEST(index_supervisor_cancel_precedes_resource_probe) { .rss_status = CBM_PROC_TREE_RSS_OK, }; cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, - index_supervisor_fake_rss, &fake); + index_supervisor_fake_charged_rss, &fake); cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 1000); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); fake.now_ms = 2000; bool cancel_accepted = handle && cbm_index_worker_request_cancel(handle); const cbm_index_worker_result_t *result = NULL; @@ -1242,11 +1297,11 @@ TEST(index_supervisor_three_failed_rss_probes_fail_closed) { .rss_status = CBM_PROC_TREE_RSS_ERROR, }; cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, - index_supervisor_fake_rss, &fake); + index_supervisor_fake_charged_rss, &fake); cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); const cbm_index_worker_result_t *result = NULL; cbm_index_worker_poll_t first = handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; @@ -1294,8 +1349,9 @@ SUITE(index_supervisor) { RUN_TEST(index_supervisor_clean_exit_without_response_is_named_failure_issue1300); RUN_TEST(index_supervisor_killed_worker_log_is_never_empty_and_names_the_run); RUN_TEST(index_supervisor_disabled_limits_do_not_probe); - RUN_TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree); + RUN_TEST(index_supervisor_charged_rss_equality_runs_then_excess_terminates_tree); RUN_TEST(index_supervisor_duration_is_total_time_not_quiet_timeout); + RUN_TEST(index_supervisor_duration_spans_recovery_attempts); RUN_TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled); RUN_TEST(index_supervisor_cancel_precedes_resource_probe); RUN_TEST(index_supervisor_three_failed_rss_probes_fail_closed); From 09dc36d26100236bc58969fda149f2c8b55bb1cb Mon Sep 17 00:00:00 2001 From: harshitaajoshi Date: Sun, 27 Sep 2026 18:16:16 -0500 Subject: [PATCH 10/27] fix(extract): parse PHP past inline HTML so symbols after ?> reach the graph (#2000) Signed-off-by: harshitaajoshi --- Makefile.cbm | 1 + internal/cbm/cbm.c | 14 ++ internal/cbm/cbm.h | 4 + internal/cbm/lsp/php_lsp.c | 4 + internal/cbm/php_inline_html.c | 317 +++++++++++++++++++++++++++++++++ tests/test_extraction.c | 128 +++++++++++++ tests/test_pipeline.c | 60 +++++++ 7 files changed, 528 insertions(+) create mode 100644 internal/cbm/php_inline_html.c diff --git a/Makefile.cbm b/Makefile.cbm index 6d19f430ae..fec175f675 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -398,6 +398,7 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/route_path.c \ $(CBM_DIR)/openapi.c \ $(CBM_DIR)/sql_values.c \ + $(CBM_DIR)/php_inline_html.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/cpp_branch_views.c \ $(CBM_DIR)/result_compact.c \ diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 23b871dbc0..4eb030ae0d 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -3313,6 +3313,18 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage return r; } +/* The bytes the grammar parses. PHP that leaves PHP mode (`?> markup ` (#2000). Same length and line breaks as the input; returns +// `source` itself when nothing needs rewriting. Defined in php_inline_html.c. +const char *cbm_php_mask_inline_html(CBMArena *arena, const char *source, int source_len); void cbm_extract_imports(CBMExtractCtx *ctx); void cbm_extract_usages(CBMExtractCtx *ctx); void cbm_extract_semantic(CBMExtractCtx *ctx); diff --git a/internal/cbm/lsp/php_lsp.c b/internal/cbm/lsp/php_lsp.c index 7aae329e82..85118f671c 100644 --- a/internal/cbm/lsp/php_lsp.c +++ b/internal/cbm/lsp/php_lsp.c @@ -20,6 +20,7 @@ #include "php_lsp.h" #include "lsp_node_iter.h" #include "lsp_work.h" +#include "../cbm.h" #include "../helpers.h" #include "../../../src/foundation/hash_table.h" #include @@ -4795,6 +4796,9 @@ void cbm_run_php_lsp_cross_with_bindings(CBMArena *arena, const char *source, in const CBMPHPImportBindings *bindings) { if (!arena || !source || source_len <= 0 || !out) return; + /* The retained source is the file on disk; read it the way the per-file + * extract did, so the cached tree and node text agree (#2000). */ + source = cbm_php_mask_inline_html(arena, source, source_len); TSParser *parser = NULL; TSTree *tree = cached_tree; diff --git a/internal/cbm/php_inline_html.c b/internal/cbm/php_inline_html.c new file mode 100644 index 0000000000..df5652d542 --- /dev/null +++ b/internal/cbm/php_inline_html.c @@ -0,0 +1,317 @@ +// php_inline_html.c — let the php_only grammar read a file that leaves PHP mode. +// +// A PHP file starts in inline-HTML mode and may switch in and out of PHP any +// number of times: `?>
...
` or for the +// text that follows it, so every byte after the first close tag used to land in +// an ERROR region and the declarations there never reached the graph (#2000). +// +// Instead of parsing the markup, blank it. The rewritten buffer has the same +// length as the original and keeps every line break, so byte offsets, rows and +// columns are unchanged and node text inside PHP code is byte-identical: +// +// - markup before the first open tag and between tags becomes spaces +// - the first open tag stays, so the tree keeps its php_tag +// - each later `` becomes `; `, the statement terminator PHP itself treats it as +// +// Only code and `//` / `#` comments can close PHP mode. Strings, heredoc, +// nowdoc and block comments are skipped, `#[` is an attribute, and everything +// after __halt_compiler is left alone, as PHP stops reading there. + +#include "cbm.h" +#include "foundation/arena.h" +#include "foundation/constants.h" +#include +#include +#include + +enum { + PHP_TAG_OPEN_LEN = 5, /* = 0 && i <= len && (size_t)(len - i) >= n && memcmp(s + i, lit, n) == 0; +} + +static bool php_is_ident_char(unsigned char c) { + return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c >= PHP_FIRST_NON_ASCII; +} + +static bool php_is_ident_start(unsigned char c) { + return php_is_ident_char(c) && !(c >= '0' && c <= '9'); +} + +static bool php_is_space(unsigned char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; +} + +static bool php_is_blank(char c) { + return c == ' ' || c == '\t'; +} + +/* Case-insensitive match of the lowercase `word` at s[i]. */ +static bool php_ieq(const char *s, int len, int i, const char *word) { + int n = (int)strlen(word); + if (i + n > len) { + return false; + } + for (int k = 0; k < n; k++) { + char c = s[i + k]; + if (c >= 'A' && c <= 'Z') { + c = (char)(c - 'A' + 'a'); + } + if (c != word[k]) { + return false; + } + } + return true; +} + +/* Length of the open tag at i, or 0. `` prolog in markup stays markup. */ +static int php_open_tag_len(const char *s, int len, int i) { + if (!php_at(s, len, i, "")) { + return true; + } + } + return false; +} + +/* Blank [from, to) in place, keeping line breaks. */ +static void php_blank(char *out, int from, int to) { + for (int k = from; k < to; k++) { + if (out[k] != '\n' && out[k] != '\r') { + out[k] = ' '; + } + } +} + +/* Index just past the quoted string opened at i. */ +static int php_skip_quoted(const char *s, int len, int i) { + char quote = s[i]; + i += SKIP_ONE; + while (i < len && s[i] != quote) { + i += (s[i] == '\\') ? PAIR_LEN : SKIP_ONE; + } + return i < len ? i + SKIP_ONE : len; +} + +/* Index just past the block comment opened at i. */ +static int php_skip_block_comment(const char *s, int len, int i) { + for (int k = i + PAIR_LEN; k < len; k++) { + if (php_at(s, len, k, "*/")) { + return k + PAIR_LEN; + } + } + return len; +} + +/* Index of the line break or `?>` that ends the one-line comment at i. */ +static int php_skip_line_comment(const char *s, int len, int i) { + while (i < len && s[i] != '\n' && !php_at(s, len, i, "?>")) { + i++; + } + return i; +} + +/* Parse the heredoc/nowdoc opener `<<= len || s[p] != quote) { + return PHP_NOT_FOUND; + } + p++; + } + if (p < len && s[p] == '\r') { + p++; + } + return (p < len && s[p] == '\n') ? p : PHP_NOT_FOUND; +} + +/* True when the line starting at p closes the heredoc. PHP 7.3+ allows the + * closing label to be indented and followed by any non-identifier byte. */ +static bool php_heredoc_closes(const char *s, int len, int p, int label, int label_len, int *end) { + while (p < len && php_is_blank(s[p])) { + p++; + } + if (len - p < label_len || memcmp(s + p, s + label, (size_t)label_len) != 0) { + return false; + } + int after = p + label_len; + if (after < len && php_is_ident_char((unsigned char)s[after])) { + return false; + } + *end = after; + return true; +} + +/* Index just past the heredoc/nowdoc opened by the `<<<` at i. */ +static int php_skip_heredoc(const char *s, int len, int i) { + int label = 0; + int label_len = 0; + int nl = php_heredoc_opener(s, len, i, &label, &label_len); + if (nl < 0) { + return i + PHP_HEREDOC_LEN; + } + while (nl < len) { + int line = nl + SKIP_ONE; + int end = 0; + if (php_heredoc_closes(s, len, line, label, label_len, &end)) { + return end; + } + const char *next = memchr(s + line, '\n', (size_t)(len - line)); + if (!next) { + return len; + } + nl = (int)(next - s); + } + return len; +} + +/* Index just past the identifier at i; sets *halt when it is __halt_compiler + * (a keyword, so not a `$variable` of that name). */ +static int php_skip_ident(const char *s, int len, int i, bool *halt) { + int start = i; + while (i < len && php_is_ident_char((unsigned char)s[i])) { + i++; + } + bool is_var = start > 0 && s[start - SKIP_ONE] == '$'; + *halt = + !is_var && i - start == PHP_HALT_COMPILER_LEN && php_ieq(s, len, start, PHP_HALT_COMPILER); + return i; +} + +/* Scan PHP code from i to the next close tag. Returns the index of that `?>`, + * len when the file ends in PHP mode, or PHP_NOT_FOUND at __halt_compiler. */ +static int php_scan_code(const char *s, int len, int i) { + while (i < len) { + unsigned char c = (unsigned char)s[i]; + if (php_at(s, len, i, "?>")) { + return i; + } + if (c == '\'' || c == '"' || c == '`') { + i = php_skip_quoted(s, len, i); + } else if (php_at(s, len, i, "/*")) { + i = php_skip_block_comment(s, len, i); + } else if (php_at(s, len, i, "//") || (c == '#' && !php_at(s, len, i, "#["))) { + i = php_skip_line_comment(s, len, i); + } else if (php_at(s, len, i, "<<<")) { + i = php_skip_heredoc(s, len, i); + } else if (php_is_ident_start(c)) { + bool halt = false; + i = php_skip_ident(s, len, i, &halt); + if (halt) { + return PHP_NOT_FOUND; + } + } else { + i++; + } + } + return len; +} + +const char *cbm_php_mask_inline_html(CBMArena *arena, const char *source, int source_len) { + if (!arena || !source || source_len <= 0) { + return source; + } + /* Fast path: a file that opens PHP at its first byte and never writes a + * close tag is exactly what the grammar already accepts. */ + if (php_open_tag_len(source, source_len, 0) > 0 && !php_has_close_tag(source, source_len)) { + return source; + } + + char *out = (char *)cbm_arena_alloc(arena, (size_t)source_len + SKIP_ONE); + if (!out) { + return source; + } + memcpy(out, source, (size_t)source_len); + out[source_len] = '\0'; + + bool changed = false; + bool seen_open = false; + int i = 0; + while (i < source_len) { + /* Inline-HTML mode: blank up to the next open tag. */ + int html_start = i; + int tag = 0; + while (i < source_len && (tag = php_open_tag_len(source, source_len, i)) == 0) { + i++; + } + if (i > html_start) { + php_blank(out, html_start, i); + changed = true; + } + if (i >= source_len) { + break; + } + if (seen_open) { + php_blank(out, i, i + tag); + changed = true; + } + seen_open = true; + + /* PHP mode, up to the close tag. */ + int close = php_scan_code(source, source_len, i + tag); + if (close < 0 || close >= source_len) { + break; + } + out[close] = ';'; + out[close + SKIP_ONE] = ' '; + changed = true; + i = close + PAIR_LEN; + } + return changed ? out : source; +} diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 7bfda63fca..c8d47ca520 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -833,6 +833,130 @@ TEST(php_function) { PASS(); } +/* #2000: inline HTML between `?>` and `` fell into an + * ERROR region and never reached the graph. */ +static const CBMCall *find_call_named(CBMFileResult *r, const char *callee) { + for (int i = 0; i < r->calls.count; i++) { + if (r->calls.items[i].callee_name && strcmp(r->calls.items[i].callee_name, callee) == 0) + return &r->calls.items[i]; + } + return NULL; +} + +static int php_def_line(CBMFileResult *r, const char *name) { + for (int i = 0; i < r->defs.count; i++) { + if (strcmp(r->defs.items[i].name, name) == 0) + return (int)r->defs.items[i].start_line; + } + return -1; +} + +TEST(php_inline_html_tail_reaches_graph_issue2000) { + CBMFileResult *r = extract("\n" + "
\n" + "

plain markup after the closing tag

\n" + "
\n" + "has_error); + ASSERT(has_def(r, "Function", "gamma")); + ASSERT(has_def(r, "Function", "delta")); + /* Lines are in original-file coordinates. */ + ASSERT_EQ(php_def_line(r, "gamma"), 2); + ASSERT_EQ(php_def_line(r, "delta"), 8); + const CBMCall *c = find_call_named(r, "gamma"); + ASSERT_NOT_NULL(c); + ASSERT_NOT_NULL(strstr(c->enclosing_func_qn, "delta")); + ASSERT_FALSE(r->parse_incomplete); + cbm_free_result(r); + PASS(); +} + +/* View-file shapes: echo tags keep their calls, and control structures that + * are opened in one PHP block and closed in a later one still parse. */ +TEST(php_inline_html_template_shapes_issue2000) { + CBMFileResult *r = extract("\n" + "<?= page_title($page) ?>\n" + "\n" + "\n" + "

Hello name?>

\n" + "\n" + " log in\n" + "\n" + "\n" + "
    • \n" + "\n" + "\n" + "\n", + CBM_LANG_PHP, "t", "view.php"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_call(r, "page_title")); + ASSERT(has_call(r, "render_item")); + ASSERT(has_def(r, "Function", "footer_links")); + ASSERT_EQ(php_def_line(r, "footer_links"), 12); + ASSERT(has_call(r, "nav_links")); + ASSERT_FALSE(r->parse_incomplete); + cbm_free_result(r); + PASS(); +} + +/* `?>` only leaves PHP mode from code or a one-line comment. Inside strings, + * heredoc, nowdoc and block comments it is text; `#[` is an attribute, not a + * comment; ` quoted';\n" + "$b = \"double ?> quoted \\\" still\";\n" + "$c = << body\n" + " EOT;\n" + "$d = <<<'NOW'\n" + "nowdoc ?> body\n" + "NOW;\n" + "/* block ?> comment */\n" + "#[Pure]\n" + "function one() { return 1; }\n" + "// line comment ends here ?>\n" + "\n" + "has_error); + ASSERT(has_def(r, "Function", "one")); + ASSERT(has_def(r, "Function", "two")); + ASSERT_EQ(php_def_line(r, "two"), 16); + ASSERT(has_call(r, "one")); + ASSERT_FALSE(r->parse_incomplete); + cbm_free_result(r); + PASS(); +} + +/* A trailing `?>` (common at the end of older class files) is not a partial + * parse, and CRLF line endings keep their line numbers. */ +TEST(php_inline_html_trailing_close_tag_crlf_issue2000) { + CBMFileResult *r = extract("\r\n", + CBM_LANG_PHP, "t", "Repo.php"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_def(r, "Class", "Repo")); + ASSERT(has_def(r, "Method", "find")); + ASSERT_EQ(php_def_line(r, "find"), 3); + ASSERT_FALSE(r->parse_incomplete); + cbm_free_result(r); + PASS(); +} + /* --- Ruby --- */ TEST(ruby_class) { CBMFileResult *r = extract("class Animal\n def initialize(name)\n @name = name\n end\n " @@ -13125,6 +13249,10 @@ SUITE(extraction) { RUN_TEST(python_class_base_extracted_bare); RUN_TEST(php_class); RUN_TEST(php_function); + RUN_TEST(php_inline_html_tail_reaches_graph_issue2000); + RUN_TEST(php_inline_html_template_shapes_issue2000); + RUN_TEST(php_inline_html_close_tag_in_literals_issue2000); + RUN_TEST(php_inline_html_trailing_close_tag_crlf_issue2000); RUN_TEST(ruby_class); RUN_TEST(ruby_module); RUN_TEST(csharp_class); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index d9b3272263..94a281a076 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -1376,6 +1376,65 @@ TEST(pipeline_nix_scoped_binding_calls_resolve) { PASS(); } +/* #2000, end to end: functions declared after an inline-HTML block reach the + * store with their CALLS edges, in the same file and across files. */ +TEST(pipeline_php_inline_html_calls_resolve_issue2000) { + if (setup_test_repo() != 0) { + FAIL("failed to create temp dir"); + } + + static const struct { + const char *name; + const char *body; + } files[] = { + {"inline_html.php", "\n" + "
    \n" + "

    plain markup after the closing tag

    \n" + "
    \n" + "\n" + "\n" + "

    \n" + " Date: Sun, 27 Sep 2026 18:55:55 -0500 Subject: [PATCH 11/27] fix(extract): address review on PHP inline-HTML masking (#2000) Signed-off-by: harshitaajoshi --- internal/cbm/php_inline_html.c | 65 +++++++++++++++++++++++++--------- tests/test_extraction.c | 52 +++++++++++++++++++++++++++ tests/test_php_lsp.c | 60 +++++++++++++++++++++++++++++++ tests/test_pipeline.c | 10 +----- 4 files changed, 161 insertions(+), 26 deletions(-) diff --git a/internal/cbm/php_inline_html.c b/internal/cbm/php_inline_html.c index df5652d542..e766a1fbdc 100644 --- a/internal/cbm/php_inline_html.c +++ b/internal/cbm/php_inline_html.c @@ -263,6 +263,47 @@ static int php_scan_code(const char *s, int len, int i) { return len; } +/* The rewritten buffer. It is allocated and filled from the source on the + * first write, so a file whose only `?>` bytes sit in strings or comments + * costs no copy. */ +typedef struct { + CBMArena *arena; + const char *source; + int len; + char *out; + bool failed; +} PHPMask; + +static char *php_mask_writable(PHPMask *m) { + if (!m->out && !m->failed) { + m->out = (char *)cbm_arena_alloc(m->arena, (size_t)m->len + SKIP_ONE); + if (m->out) { + memcpy(m->out, m->source, (size_t)m->len); + m->out[m->len] = '\0'; + } else { + m->failed = true; + } + } + return m->out; +} + +/* Blank [from, to) of the rewritten buffer, keeping line breaks. */ +static void php_mask_blank(PHPMask *m, int from, int to) { + char *out = php_mask_writable(m); + if (out) { + php_blank(out, from, to); + } +} + +/* Turn the `?>` at i into `; `. */ +static void php_mask_close_tag(PHPMask *m, int i) { + char *out = php_mask_writable(m); + if (out) { + out[i] = ';'; + out[i + SKIP_ONE] = ' '; + } +} + const char *cbm_php_mask_inline_html(CBMArena *arena, const char *source, int source_len) { if (!arena || !source || source_len <= 0) { return source; @@ -273,17 +314,10 @@ const char *cbm_php_mask_inline_html(CBMArena *arena, const char *source, int so return source; } - char *out = (char *)cbm_arena_alloc(arena, (size_t)source_len + SKIP_ONE); - if (!out) { - return source; - } - memcpy(out, source, (size_t)source_len); - out[source_len] = '\0'; - - bool changed = false; + PHPMask m = {.arena = arena, .source = source, .len = source_len}; bool seen_open = false; int i = 0; - while (i < source_len) { + while (i < source_len && !m.failed) { /* Inline-HTML mode: blank up to the next open tag. */ int html_start = i; int tag = 0; @@ -291,15 +325,13 @@ const char *cbm_php_mask_inline_html(CBMArena *arena, const char *source, int so i++; } if (i > html_start) { - php_blank(out, html_start, i); - changed = true; + php_mask_blank(&m, html_start, i); } if (i >= source_len) { break; } if (seen_open) { - php_blank(out, i, i + tag); - changed = true; + php_mask_blank(&m, i, i + tag); } seen_open = true; @@ -308,10 +340,9 @@ const char *cbm_php_mask_inline_html(CBMArena *arena, const char *source, int so if (close < 0 || close >= source_len) { break; } - out[close] = ';'; - out[close + SKIP_ONE] = ' '; - changed = true; + php_mask_close_tag(&m, close); i = close + PAIR_LEN; } - return changed ? out : source; + /* On allocation failure parse the file as it is: the old behaviour. */ + return (m.out && !m.failed) ? m.out : source; } diff --git a/tests/test_extraction.c b/tests/test_extraction.c index c8d47ca520..4d03423c1d 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -938,6 +938,56 @@ TEST(php_inline_html_close_tag_in_literals_issue2000) { PASS(); } +/* Markup is blanked, never parsed: code-shaped text outside the PHP tags must + * not produce definitions or calls, wherever it sits. */ +TEST(php_inline_html_markup_never_reaches_graph_issue2000) { + CBMFileResult *r = extract("

    function before() { return leakBefore(); }

    \n" + "\n" + "

    function fake() { return leak(); }

    \n" + "\n" + "function xmlFake() { leakXml(); }\n" + "\n" + "

    class Tail { function tailFake() { leakTail(); } }

    \n", + CBM_LANG_PHP, "t", "markup.php"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_def(r, "Function", "real")); + ASSERT(has_def(r, "Function", "after")); + ASSERT(has_call(r, "real")); + ASSERT_FALSE(has_def_any(r, "before")); + ASSERT_FALSE(has_def_any(r, "fake")); + ASSERT_FALSE(has_def_any(r, "jsFake")); + ASSERT_FALSE(has_def_any(r, "xmlFake")); + ASSERT_FALSE(has_def_any(r, "Tail")); + ASSERT_FALSE(has_def_any(r, "tailFake")); + ASSERT_FALSE(has_call(r, "leak")); + ASSERT_FALSE(r->parse_incomplete); + cbm_free_result(r); + PASS(); +} + +/* The mask hands back the caller's buffer, uncopied, when nothing needs + * rewriting, and a same-length rewrite otherwise. */ +TEST(php_inline_html_mask_copies_only_on_write_issue2000) { + CBMArena arena; + cbm_arena_init(&arena); + const char *plain = "';\n/* ?> */\n$h = <<\nEOT;\n"; + const char *mixed = "\nx\n"; + ASSERT(cbm_php_mask_inline_html(&arena, plain, (int)strlen(plain)) == plain); + ASSERT(cbm_php_mask_inline_html(&arena, quoted, (int)strlen(quoted)) == quoted); + const char *masked = cbm_php_mask_inline_html(&arena, mixed, (int)strlen(mixed)); + ASSERT(masked != mixed); + ASSERT_EQ((int)strlen(masked), (int)strlen(mixed)); + ASSERT_STR_EQ(masked, "` (common at the end of older class files) is not a partial * parse, and CRLF line endings keep their line numbers. */ TEST(php_inline_html_trailing_close_tag_crlf_issue2000) { @@ -13252,6 +13302,8 @@ SUITE(extraction) { RUN_TEST(php_inline_html_tail_reaches_graph_issue2000); RUN_TEST(php_inline_html_template_shapes_issue2000); RUN_TEST(php_inline_html_close_tag_in_literals_issue2000); + RUN_TEST(php_inline_html_markup_never_reaches_graph_issue2000); + RUN_TEST(php_inline_html_mask_copies_only_on_write_issue2000); RUN_TEST(php_inline_html_trailing_close_tag_crlf_issue2000); RUN_TEST(ruby_class); RUN_TEST(ruby_module); diff --git a/tests/test_php_lsp.c b/tests/test_php_lsp.c index 04ff1001db..7c0314a270 100644 --- a/tests/test_php_lsp.c +++ b/tests/test_php_lsp.c @@ -110,6 +110,65 @@ TEST(phplsp_local_method_via_typed_param) { PASS(); } +/* #2000: with no cached tree (a result spilled under the memory budget comes + * back without one) the cross-file pass re-parses the on-disk source itself. + * It must blank inline HTML exactly as the per-file extract does, or a file's + * tail would lose cross-file resolution only when that file spilled. */ +static int find_resolved_arr(const CBMResolvedCallArray *arr, const char *callerSub, + const char *calleeSub) { + for (int i = 0; i < arr->count; i++) { + const CBMResolvedCall *rc = &arr->items[i]; + if (rc->caller_qn && strstr(rc->caller_qn, callerSub) && rc->callee_qn && + strstr(rc->callee_qn, calleeSub)) + return i; + } + return -1; +} + +TEST(phplsp_cross_reparse_reads_past_inline_html_issue2000) { + const char *src = "\n" + "
    \n" + "

    plain markup after the closing tag

    \n" + "
    \n" + "send(); }\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.main.gamma", + .short_name = "gamma", + .label = "Function", + .def_module_qn = "test.main"}, + {.qualified_name = "test.main.delta", + .short_name = "delta", + .label = "Function", + .def_module_qn = "test.main"}, + {.qualified_name = "test.mailer.Mailer", + .short_name = "Mailer", + .label = "Class", + .def_module_qn = "test.mailer"}, + {.qualified_name = "test.mailer.Mailer.send", + .short_name = "send", + .label = "Method", + .def_module_qn = "test.mailer", + .receiver_type = "test.mailer.Mailer"}, + }; + const char *imp_names[] = {"Mailer"}; + const char *imp_qns[] = {"test.mailer.Mailer"}; + + CBMArena arena; + cbm_arena_init(&arena); + CBMResolvedCallArray out = {0}; + + cbm_run_php_lsp_cross(&arena, src, (int)strlen(src), "test.main", defs, 4, imp_names, imp_qns, + 1, NULL, &out); + + ASSERT_GTE(find_resolved_arr(&out, "main.delta", "Mailer.send"), 0); + + cbm_arena_destroy(&arena); + PASS(); +} + /* ── 3. Arrow function with typed parameter ───────────────────── */ TEST(phplsp_arrow_function_typed_param) { @@ -5841,6 +5900,7 @@ SUITE(php_lsp) { /* Phase 1 baseline regressions */ RUN_TEST(phplsp_local_method_via_new_assignment); RUN_TEST(phplsp_local_method_via_typed_param); + RUN_TEST(phplsp_cross_reparse_reads_past_inline_html_issue2000); RUN_TEST(phplsp_arrow_function_typed_param); RUN_TEST(phplsp_static_call_resolved); RUN_TEST(phplsp_self_and_parent); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 94a281a076..c8094efac9 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -1405,15 +1405,7 @@ TEST(pipeline_php_inline_html_calls_resolve_issue2000) { "function renderCard($item) { return formatPrice($item['cents']); }\n"}, }; for (size_t i = 0; i < sizeof(files) / sizeof(files[0]); i++) { - char path[512]; - snprintf(path, sizeof(path), "%s/%s", g_tmpdir, files[i].name); - FILE *f = fopen(path, "w"); - if (!f) { - teardown_test_repo(); - FAIL("failed to write php fixture"); - } - fputs(files[i].body, f); - fclose(f); + write_temp_file(g_tmpdir, files[i].name, files[i].body); } char db[512]; From bbdeeb61a49c91fb13665a3512b7f36a4ed7e0dd Mon Sep 17 00:00:00 2001 From: harshitaajoshi Date: Mon, 28 Sep 2026 13:10:32 -0500 Subject: [PATCH 12/27] test(extract): pin PHP mask copy-on-write via arena allocation total (#2000) Signed-off-by: harshitaajoshi --- tests/test_extraction.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 4d03423c1d..ad6ecd8630 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -971,16 +971,28 @@ TEST(php_inline_html_markup_never_reaches_graph_issue2000) { } /* The mask hands back the caller's buffer, uncopied, when nothing needs - * rewriting, and a same-length rewrite otherwise. */ + * rewriting, and a same-length rewrite otherwise. The pointer alone cannot + * tell copy-on-write from an eager copy, so the arena's allocation total is + * sampled around each call: no bytes for `plain` and `quoted`, a copy for + * `mixed`. */ TEST(php_inline_html_mask_copies_only_on_write_issue2000) { CBMArena arena; cbm_arena_init(&arena); const char *plain = "';\n/* ?> */\n$h = <<\nEOT;\n"; const char *mixed = "\nx\n"; + + size_t before = cbm_arena_total(&arena); ASSERT(cbm_php_mask_inline_html(&arena, plain, (int)strlen(plain)) == plain); + ASSERT_EQ(cbm_arena_total(&arena), before); + + before = cbm_arena_total(&arena); ASSERT(cbm_php_mask_inline_html(&arena, quoted, (int)strlen(quoted)) == quoted); + ASSERT_EQ(cbm_arena_total(&arena), before); + + before = cbm_arena_total(&arena); const char *masked = cbm_php_mask_inline_html(&arena, mixed, (int)strlen(mixed)); + ASSERT(cbm_arena_total(&arena) > before); ASSERT(masked != mixed); ASSERT_EQ((int)strlen(masked), (int)strlen(mixed)); ASSERT_STR_EQ(masked, " Date: Tue, 20 Oct 2026 19:30:58 +0200 Subject: [PATCH 13/27] fix(extraction): keep Swift initializer edges to true targets Follow-up to the review of #2395 (issue #2379). Making init/deinit Method nodes gives every type an `init` target, and the short-name registry strategies then bound initializer calls to whichever `init` was nearest. On Alamofire (bda9ed57) all 8 new CALLS edges into an initializer were false, and so were the 40 new USAGE edges. - Name init/deinit in one Swift-gated place, cbm_resolve_func_name; resolve_method_name delegates there. The language-agnostic func_name_node no longer special-cases the node kinds (the Slang grammar has an init_declaration too). - super.init(...) is emitted as .init, from the first entry of the enclosing class's inheritance clause. The class node is carried by the unified walk (enclosing_class_node, saved and restored with the other frame state) instead of found by climbing parents. Without a clause, or with an entry that does not read as a type name, there is no call. - self.init(...) forms no call. Initializer overloads share one node, so the only possible edge is a self-loop, and that is false whenever the overload it delegates to lives outside the project. - T.init(...) calls are flagged is_method, and Swift joins the weak-member guard in pass_calls.c and pass_parallel.c (in lockstep), so only qualified, same-module or import placements survive. No other Swift call carries the flag. - init and deinit are Swift keywords for cbm_is_keyword, so references such as .init(...) and Foo.init are not usages resolved by name. Tests: the exact CALLS set for the #2379 reproduction plus initializer delegation in a SwiftPM package, through the sequential and the parallel resolver; extraction tests for the superclass rewrite, the no-superclass and lower-case cases, the scope of the call flag and init references. Review nits: renamed the overload test, removed the stacked helper comment. Alamofire, main / PR head / this commit: CALLS 5472 / 5487 / 5485; CALLS into an initializer 0 / 8 (all false) / 6 (all superclass initializers); USAGE into an initializer 0 / 40 / 0. Co-authored-by: Andrew Studnicky Signed-off-by: Martin Vogel --- internal/cbm/extract_calls.c | 75 ++++++++++ internal/cbm/extract_defs.c | 20 ++- internal/cbm/extract_unified.c | 8 ++ internal/cbm/extract_unified.h | 2 + internal/cbm/helpers.c | 10 ++ src/pipeline/pass_calls.c | 6 +- src/pipeline/pass_parallel.c | 4 +- tests/test_extraction.c | 132 ++++++++++++++++- tests/test_pipeline.c | 249 +++++++++++++++++++++++++++++++++ 9 files changed, 492 insertions(+), 14 deletions(-) diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index 4eb02659c4..b5cda64291 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -4278,6 +4278,70 @@ static TSNode language_specific_callee_leaf(CBMLanguage language, TSNode expr, return terminal_callee_leaf(ctx, expr); } +/* `Base.init` for a `super.init(...)` call: Base is the first entry of the + * enclosing class's inheritance clause, where Swift requires the superclass to + * stand. The class node is carried by the walk, so no ancestor climb. NULL (no + * call) outside a class, without a clause, or when the entry does not read as + * a plain type name -- a lower-case or attributed root would pass the + * registry's receiver-chain check and bind by short name, which is the false + * edge this exists to prevent. */ +static const char *swift_superclass_init_callee(CBMExtractCtx *ctx, const WalkState *state) { + TSNode cls = state ? state->enclosing_class_node : (TSNode){0}; + if (ts_node_is_null(cls) || strcmp(ts_node_type(cls), "class_declaration") != 0) { + return NULL; + } + TSNode spec = cbm_find_child_by_kind(cls, "inheritance_specifier"); + char *base = ts_node_is_null(spec) ? NULL : cbm_node_text(ctx->arena, spec, ctx->source); + if (!base || base[0] < 'A' || base[0] > 'Z') { + return NULL; + } + size_t len = strcspn(base, "<"); /* Base names Base */ + for (size_t i = 0; i < len; i++) { + if (!isalnum((unsigned char)base[i]) && base[i] != '_' && base[i] != '.') { + return NULL; + } + } + base[len] = '\0'; + return cbm_arena_sprintf(ctx->arena, "%s.init", base); +} + +/* True for a Swift initializer callee `T.init` (T non-empty). */ +static bool swift_callee_is_initializer(const char *callee) { + static const char suffix[] = ".init"; + size_t n = callee ? strlen(callee) : 0; + return n > sizeof(suffix) - 1 && strcmp(callee + n - (sizeof(suffix) - 1), suffix) == 0; +} + +/* Swift initializer delegation (#2379). `super.init(...)` and `self.init(...)` + * name no type, and a lower-case receiver passes the registry's receiver-chain + * check, so a short-name match binds them to whichever project `init` is + * nearest: a nested type's, a sibling type's, or the caller's own. On + * Alamofire all 8 such edges were false. + * + * `super.init` is rewritten to name the superclass (`Base.init`), so the + * qualified strategies place it on the project's Base.init. When Base is not + * in the project (UIViewController, NSObject) nothing places it, and the + * initializer guard in handle_calls keeps a short-name guess from doing so. + * + * `self.init` delegates to another initializer of the caller's own type. + * Initializer overloads share one node (#2061), so the only edge it could form + * is a self-loop, and that one is false whenever the overload it calls lives + * outside the project (`extension Notification { init(...) { self.init(...) } }` + * calls Foundation's). It forms no CALLS edge. */ +static const char *resolve_swift_delegating_init(CBMExtractCtx *ctx, const WalkState *state, + const char *callee) { + if (ctx->language != CBM_LANG_SWIFT || !callee) { + return callee; + } + if (strcmp(callee, "self.init") == 0) { + return NULL; + } + if (strcmp(callee, "super.init") == 0) { + return swift_superclass_init_callee(ctx, state); + } + return callee; +} + // Produce the textual callee and the AST occurrence from one operation. The // generic fallback retains the former field/terminal behavior, while grammars // whose extractor uses a sibling, a special field, or a synthetic literal are @@ -4288,6 +4352,7 @@ static CBMPrimaryCalleeSelection select_primary_callee(CBMExtractCtx *ctx, TSNod CBMPrimaryCalleeSelection selection = {0}; selection.name = extract_callee_name(ctx->arena, node, ctx->source, ctx->language); selection.name = resolve_objectscript_callee(ctx, node, state, selection.name); + selection.name = resolve_swift_delegating_init(ctx, state, selection.name); if (!selection.name || !selection.name[0]) { return selection; } @@ -4511,6 +4576,16 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML call.is_method = true; } } + // Swift initializer guard (#2379; same direction). Flag `T.init(...)`, + // as written or as a rewritten super.init. Nearly every type has an + // `init`, so a short-name match onto one is a guess: with a project + // module imported, `UIViewController.init` bound an unrelated + // project initializer. The registry's qualified / same-module / + // import strategies still place T.init when T is a project type. + // No other Swift call carries the flag. + if (ctx->language == CBM_LANG_SWIFT && swift_callee_is_initializer(call.callee_name)) { + call.is_method = true; + } TSNode args = ts_node_child_by_field_name(node, TS_FIELD("arguments")); /* tree-sitter-elixir attaches NO field name to a call's arguments diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 865f56aa73..6121613486 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -236,11 +236,6 @@ static void extract_elixir_call(CBMExtractCtx *ctx, TSNode node, const CBMLangSp // Get "name" field from a node static TSNode func_name_node(TSNode node) { - const char *node_type = ts_node_type(node); - if (strcmp(node_type, "init_declaration") == 0 || - strcmp(node_type, "deinit_declaration") == 0) { - return node; - } TSNode name = ts_node_child_by_field_name(node, TS_FIELD("name")); if (ts_node_is_null(name)) { /* Protobuf rpc: name is in rpc_name child, not "name" field */ @@ -774,6 +769,15 @@ TSNode cbm_resolve_func_name(TSNode node, CBMLanguage lang) { return cbm_find_child_by_kind(node, "query_name"); } + /* Swift init/deinit (#2379) have no name node at all. The declaration + * itself stands in for it, and cbm_func_name_node_text maps that + * node kind to "init" / "deinit". Gated on Swift: Slang's grammar + * also has an init_declaration node. */ + if (lang == CBM_LANG_SWIFT && + (strcmp(kind, "init_declaration") == 0 || strcmp(kind, "deinit_declaration") == 0)) { + return node; + } + TSNode name = func_name_node(node); if (lang == CBM_LANG_R && strcmp(kind, "function_definition") == 0) { @@ -8674,6 +8678,12 @@ static TSNode resolve_method_name(TSNode child, CBMLanguage lang) { return cbm_find_child_by_kind(child, "simple_identifier"); } + // Swift init/deinit: named in one place, cbm_resolve_func_name (#2379). + if (lang == CBM_LANG_SWIFT && + (strcmp(ck, "init_declaration") == 0 || strcmp(ck, "deinit_declaration") == 0)) { + return cbm_resolve_func_name(child, lang); + } + // Squirrel: function_declaration's name is a plain `identifier` child. if (lang == CBM_LANG_SQUIRREL && strcmp(ck, "function_declaration") == 0) { return cbm_find_child_by_kind(child, "identifier"); diff --git a/internal/cbm/extract_unified.c b/internal/cbm/extract_unified.c index fbb7e12c3b..f0163d4ffd 100644 --- a/internal/cbm/extract_unified.c +++ b/internal/cbm/extract_unified.c @@ -324,6 +324,7 @@ static bool push_scope(WalkState *state, uint8_t kind, uint32_t depth, const cha * CBMWalkScope comment for the quadratic recompute this replaces. */ f->prev_enclosing_func_qn = state->enclosing_func_qn; f->prev_enclosing_class_qn = state->enclosing_class_qn; + f->prev_enclosing_class_node = state->enclosing_class_node; f->prev_invocation_kind = state->invocation_kind; f->prev_callee_expr = state->callee_expr; f->prev_callee_leaf = state->callee_leaf; @@ -337,6 +338,8 @@ static bool push_scope(WalkState *state, uint8_t kind, uint32_t depth, const cha case SCOPE_CLASS: case SCOPE_NAMESPACE: state->enclosing_class_qn = qn; + /* push_lexical_scope records the node; a node-less push has none. */ + state->enclosing_class_node = (TSNode){0}; break; case SCOPE_CALL: /* The invocation triple is applied by push_call_scope once the caller @@ -364,6 +367,9 @@ static bool push_lexical_scope(WalkState *state, uint8_t walk_kind, uint32_t dep state->lexical_binding_tracking_failed = true; return false; } + if (walk_kind == SCOPE_CLASS || walk_kind == SCOPE_NAMESPACE) { + state->enclosing_class_node = node; + } state->scopes[state->scope_top - SKIP_ONE].lexical_scope_id = add_lexical_scope(state, node, lexical_kind); return true; @@ -442,6 +448,7 @@ static void pop_expired_scopes(WalkState *state, uint32_t cur_depth) { const CBMWalkScope *f = &state->scopes[--state->scope_top]; state->enclosing_func_qn = f->prev_enclosing_func_qn; state->enclosing_class_qn = f->prev_enclosing_class_qn; + state->enclosing_class_node = f->prev_enclosing_class_node; state->invocation_kind = f->prev_invocation_kind; state->callee_expr = f->prev_callee_expr; state->callee_leaf = f->prev_callee_leaf; @@ -2671,6 +2678,7 @@ void cbm_extract_unified(CBMExtractCtx *ctx) { * recompute_state call): module scope, nothing else active. */ state.enclosing_func_qn = ctx->module_qn; state.enclosing_class_qn = NULL; + state.enclosing_class_node = (TSNode){0}; state.invocation_kind = CBM_INVOCATION_NONE; state.callee_expr = (TSNode){0}; state.callee_leaf = (TSNode){0}; diff --git a/internal/cbm/extract_unified.h b/internal/cbm/extract_unified.h index f80237229d..8e40d6a2cf 100644 --- a/internal/cbm/extract_unified.h +++ b/internal/cbm/extract_unified.h @@ -78,6 +78,7 @@ typedef struct { * suite-budget kills on every non-M4 venue). */ const char *prev_enclosing_func_qn; const char *prev_enclosing_class_qn; + TSNode prev_enclosing_class_node; CBMInvocationKind prev_invocation_kind; TSNode prev_callee_expr; TSNode prev_callee_leaf; @@ -133,6 +134,7 @@ typedef struct { typedef struct { const char *enclosing_func_qn; // current function QN (module_qn at top level) const char *enclosing_class_qn; // current class QN (NULL outside class) + TSNode enclosing_class_node; // that class scope's node (null when not known) const TSTreeCursor *current_cursor; // unified walk cursor at the current node TSTreeCursor *occurrence_cursor; // reusable parent-preserving classifier cursor CBMInvocationKind invocation_kind; // exact active invocation/reference role diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index f03c0c940c..53b12d44cd 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -301,6 +301,16 @@ bool cbm_is_keyword(const char *name, CBMLanguage lang) { case CBM_LANG_PUPPET: keywords = puppet_keywords; break; + case CBM_LANG_SWIFT: + /* `init` and `deinit` are Swift keywords. A reference to one + * (`.init(...)`, `Foo.init`) names an initializer of a type the bare + * name does not carry, so resolving it by name can only guess among + * every type's initializer (#2379). Otherwise the generic set. */ + if (strcmp(name, "init") == 0 || strcmp(name, "deinit") == 0) { + return true; + } + keywords = generic_keywords; + break; default: keywords = generic_keywords; break; diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index 738e5b9328..b41530cba7 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -1133,7 +1133,11 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM * to a docs bundle (2026-09-16 probe: 4,207 junk * edges on JetBrains/Exposed). */ lang == CBM_LANG_HTML || lang == CBM_LANG_VUE || - lang == CBM_LANG_SVELTE || lang == CBM_LANG_ASTRO; + lang == CBM_LANG_SVELTE || lang == CBM_LANG_ASTRO || + /* Swift flags only initializer calls `T.init(...)` + * (#2379, extract_calls.c); every other Swift call + * keeps is_method=false and is unaffected. */ + lang == CBM_LANG_SWIFT; /* Bare-call local-binding suppression. A member call has a receiver the * guard above can reason about; a bare `run()` has none, so that guard * cannot see this class at all. Python-only today because the extraction diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 302e76faa4..0f71f38d3e 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -3401,7 +3401,9 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB lang == CBM_LANG_ARKTS || /* embedded-script hosts — see pass_calls.c */ lang == CBM_LANG_HTML || lang == CBM_LANG_VUE || - lang == CBM_LANG_SVELTE || lang == CBM_LANG_ASTRO; + lang == CBM_LANG_SVELTE || lang == CBM_LANG_ASTRO || + /* Swift initializer calls only — see pass_calls.c */ + lang == CBM_LANG_SWIFT; /* Bare-call local-binding suppression — see the note in pass_calls.c. * This gate MUST stay identical to the one there. */ bool suppress_weak_local_binding = lang == CBM_LANG_PYTHON; diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 83b888ec62..42c4654317 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -79,8 +79,6 @@ static int has_def_qn(CBMFileResult *r, const char *qn) { return 0; } -/* Return the definition with the given qualified name, or NULL. Distinct from - * find_def_by_name for the same reason has_def_qn is distinct from has_def. */ /* Find a definition by its qualified name relative to the file's module QN * (e.g. "Box.init" resolves to ".Box.init"). */ static const CBMDefinition *find_def_in_module(CBMFileResult *r, const char *rel_qn) { @@ -2599,9 +2597,11 @@ TEST(swift_init_convenience_delegates_issue2379) { const CBMDefinition *init_def = find_def_in_module(r, "Holder.init"); ASSERT_NOT_NULL(init_def); - /* self.init(...) is a plain member call today, same as any other - * `receiver.member(...)` invocation -- no dedicated delegation handling. */ - ASSERT(has_call(r, "self.init")); + /* self.init(...) delegates to another overload of Holder.init, which + * shares this node (#2061): it forms no call, because the only edge it + * could produce is a self-loop. Its argument call stays. */ + ASSERT_FALSE(has_call(r, "self.init")); + ASSERT_FALSE(has_call(r, "Holder.init")); const CBMCall *call = find_call_by_callee(r, "other"); ASSERT_NOT_NULL(call); @@ -2619,7 +2619,7 @@ TEST(swift_init_convenience_delegates_issue2379) { * Swift methods onto one node (DeusData/codebase-memory-mcp#2061), and an * init is no exception; distinguishing overloads by signature is that * issue's concern, not this one's. */ -TEST(swift_init_multiple_not_collapsed_issue2379) { +TEST(swift_init_overloads_share_one_node_issue2379) { CBMFileResult *r = extract("func shared() -> Int { 3 }\n" "final class Holder {\n" " let value: Int\n" @@ -2761,6 +2761,120 @@ TEST(swift_deinit_issue2379) { PASS(); } +/* #2379: `super.init(...)` names no type. Left as written, the registry binds it + * by short name to whichever project `init` is nearest (on Alamofire: a nested + * `Options.init`, a sibling decoder's). It is emitted as the superclass's + * initializer instead -- the first inheritance entry, generic arguments + * dropped -- and stays attributed to the calling initializer. */ +TEST(swift_super_init_names_superclass_issue2379) { + CBMFileResult *r = extract("final class Sub: Base, Sendable {\n" + " override init() {\n" + " super.init()\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Sub.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *init_def = find_def_in_module(r, "Sub.init"); + ASSERT_NOT_NULL(init_def); + ASSERT_FALSE(has_call(r, "super.init")); + const CBMCall *call = find_call_by_callee(r, "Base.init"); + ASSERT_NOT_NULL(call); + ASSERT_NOT_NULL(call->enclosing_func_qn); + ASSERT_STR_EQ(call->enclosing_func_qn, init_def->qualified_name); + + cbm_free_result(r); + PASS(); +} + +/* #2379: without a superclass to name -- no inheritance clause, or a first entry + * that does not read as a type -- `super.init` forms no call at all. A + * lower-case root would pass the registry's receiver-chain check and bind by + * short name again. */ +TEST(swift_super_init_without_superclass_no_call_issue2379) { + CBMFileResult *r = extract("class Root {\n" + " init() {\n" + " super.init()\n" + " }\n" + "}\n" + "class Odd: base {\n" + " init() {\n" + " super.init()\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Root.swift"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(find_def_in_module(r, "Root.init")); + ASSERT_NOT_NULL(find_def_in_module(r, "Odd.init")); + for (int i = 0; i < r->calls.count; i++) { + const char *callee = r->calls.items[i].callee_name; + ASSERT(!callee || !strstr(callee, "init")); + } + + cbm_free_result(r); + PASS(); +} + +/* #2379: an initializer call `T.init(...)` is flagged so the resolvers' + * weak-member guard keeps a short-name guess from placing it (nearly every + * type has an `init`); every other Swift call stays unflagged and resolves + * exactly as before. */ +TEST(swift_initializer_calls_flagged_issue2379) { + CBMFileResult *r = extract("func run(obj: Worker) {\n" + " _ = UIAlertAction.init(title: nil)\n" + " helper()\n" + " obj.start()\n" + "}\n", + CBM_LANG_SWIFT, "t", "Run.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + const CBMCall *init_call = find_call_by_callee(r, "UIAlertAction.init"); + const CBMCall *bare = find_call_by_callee(r, "helper"); + const CBMCall *member = find_call_by_callee(r, "obj.start"); + ASSERT_NOT_NULL(init_call); + ASSERT_NOT_NULL(bare); + ASSERT_NOT_NULL(member); + ASSERT_TRUE(init_call->is_method); + ASSERT_FALSE(bare->is_method); + ASSERT_FALSE(member->is_method); + + cbm_free_result(r); + PASS(); +} + +/* #2379: `init` is a Swift keyword. A reference to one (`.init(...)`, + * `Box.init`) names the initializer of a type the bare name does not carry, + * so it is not recorded as a usage -- by name it could only be guessed among + * every type's initializer (on Alamofire, 26 references landed on one nested + * `Output.init`). The type in `Box.init` is still a usage. */ +TEST(swift_init_reference_is_not_a_usage_issue2379) { + CBMFileResult *r = extract("struct Box {\n" + " init(raw: Int) {}\n" + "}\n" + "func make() -> Box {\n" + " .init(raw: 1)\n" + "}\n" + "func maker() -> (Int) -> Box {\n" + " Box.init\n" + "}\n", + CBM_LANG_SWIFT, "t", "Make.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + int init_usages = 0; + int box_usages = 0; + for (int i = 0; i < r->usages.count; i++) { + const char *ref = r->usages.items[i].ref_name; + init_usages += ref && strcmp(ref, "init") == 0; + box_usages += ref && strcmp(ref, "Box") == 0; + } + ASSERT_EQ(init_usages, 0); + ASSERT_GT(box_usages, 0); + + cbm_free_result(r); + PASS(); +} + /* --- Objective-C --- */ TEST(objc_interface) { CBMFileResult *r = @@ -13488,11 +13602,15 @@ SUITE(extraction) { RUN_TEST(swift_init_struct_issue2379); RUN_TEST(swift_init_class_designated_issue2379); RUN_TEST(swift_init_convenience_delegates_issue2379); - RUN_TEST(swift_init_multiple_not_collapsed_issue2379); + RUN_TEST(swift_init_overloads_share_one_node_issue2379); RUN_TEST(swift_init_failable_enum_issue2379); RUN_TEST(swift_init_extension_issue2379); RUN_TEST(swift_method_still_attributed_issue2379); RUN_TEST(swift_deinit_issue2379); + RUN_TEST(swift_super_init_names_superclass_issue2379); + RUN_TEST(swift_super_init_without_superclass_no_call_issue2379); + RUN_TEST(swift_initializer_calls_flagged_issue2379); + RUN_TEST(swift_init_reference_is_not_a_usage_issue2379); RUN_TEST(swift_call_string_arg_issue1892); RUN_TEST(swift_nested_url_constructor_issue1892); RUN_TEST(swift_nested_url_no_bang_issue1892); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 7b868a7c16..8d4aaddc6d 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -18979,6 +18979,253 @@ TEST(pipeline_swift_cross_package_import) { PASS(); } +/* #2379 at graph level, as a SwiftPM package. Sources/Kit/Inits.swift is the + * issue's reproduction. Sources/App/Sub.swift imports the Kit target -- which + * sends name resolution down the import-aware path -- and calls initializers + * the way real code does: `super.init` to a project superclass in another + * target, `super.init` to a superclass outside the project, an explicit + * `T.init(...)` on a project type and on a type outside the project, `self.init` + * from an extension of a type outside the project, and an implicit `.init(...)`. + * Each of those can attract a false CALLS edge now that initializers are nodes. */ +static const char *const SWIFT_INIT_PACKAGE_SRC = + "let package = Package(\n" + " name: \"Kit\",\n" + " products: [.library(name: \"Kit\", targets: [\"Kit\"])],\n" + " targets: [\n" + " .target(name: \"Kit\", dependencies: []),\n" + " .target(name: \"App\", dependencies: [\"Kit\"])\n" + " ]\n" + ")\n"; + +static const char *const SWIFT_INIT_INITS_SRC = "func helper() -> Int { 1 }\n" + "func other() -> Int { 2 }\n" + "\n" + "struct Box {\n" + " let value: Int\n" + " init() {\n" + " value = helper()\n" + " }\n" + " func read() -> Int {\n" + " other()\n" + " }\n" + "}\n" + "\n" + "final class Holder {\n" + " let value: Int\n" + " init(seed: Int) {\n" + " value = helper() + seed\n" + " }\n" + " convenience init() {\n" + " self.init(seed: other())\n" + " }\n" + "}\n" + "\n" + "enum Mode {\n" + " case on\n" + " init?(flag: Bool) {\n" + " if helper() > 0 { self = .on } else { " + "return nil }\n" + " }\n" + "}\n" + "\n" + "extension Box {\n" + " init(raw: Int) {\n" + " value = other()\n" + " }\n" + "}\n"; + +static const char *const SWIFT_INIT_BASE_SRC = "open class Base {\n" + " public init() {}\n" + "}\n"; + +static const char *const SWIFT_INIT_SUB_SRC = "import Kit\n" + "import UIKit\n" + "\n" + "class Sub: Base {\n" + " override init() {\n" + " super.init()\n" + " }\n" + "}\n" + "class Screen: UIViewController {\n" + " init() {\n" + " super.init(nibName: nil, bundle: nil)\n" + " }\n" + " func alert() -> UIAlertAction {\n" + " UIAlertAction.init(title: nil, style: " + ".default)\n" + " }\n" + "}\n" + "extension Notification {\n" + " init(tag: String) {\n" + " self.init(name: Notification.Name(tag))\n" + " }\n" + "}\n" + "func make() -> Box {\n" + " .init(raw: 1)\n" + "}\n" + "func explicit() -> Box {\n" + " Box.init(raw: 2)\n" + "}\n"; + +enum { SWIFT_INIT_EDGE_CAP = 32, SWIFT_INIT_EDGE_LEN = 192 }; + +typedef struct { + int run_rc; + int calls; /* CALLS edges seen; -1 if the store was not read */ + int init_usages; /* USAGE edges into a node named init/deinit */ + char edges[SWIFT_INIT_EDGE_CAP][SWIFT_INIT_EDGE_LEN]; /* "src -> tgt", sorted */ +} SwiftInitObservation; + +static int swift_init_edge_cmp(const void *a, const void *b) { + return strcmp((const char *)a, (const char *)b); +} + +/* Qualified name relative to the project ("Inits.Box.init"). */ +static const char *swift_init_rel_qn(const char *qn, const char *project) { + size_t plen = strlen(project); + return strncmp(qn, project, plen) == 0 && qn[plen] == '.' ? qn + plen + 1 : qn; +} + +static void swift_init_observe(cbm_store_t *s, const char *project, SwiftInitObservation *o) { + cbm_edge_t *edges = NULL; + int count = 0; + o->calls = 0; + if (cbm_store_find_edges_by_type(s, project, "CALLS", &edges, &count) == CBM_STORE_OK) { + for (int i = 0; i < count; i++) { + cbm_node_t src = {0}; + cbm_node_t tgt = {0}; + if (cbm_store_find_node_by_id(s, edges[i].source_id, &src) == CBM_STORE_OK && + cbm_store_find_node_by_id(s, edges[i].target_id, &tgt) == CBM_STORE_OK && + o->calls < SWIFT_INIT_EDGE_CAP) { + snprintf(o->edges[o->calls], SWIFT_INIT_EDGE_LEN, "%s -> %s", + swift_init_rel_qn(src.qualified_name, project), + swift_init_rel_qn(tgt.qualified_name, project)); + } + o->calls++; + cbm_node_free_fields(&src); + cbm_node_free_fields(&tgt); + } + if (edges) { + cbm_store_free_edges(edges, count); + } + } + int kept = o->calls < SWIFT_INIT_EDGE_CAP ? o->calls : SWIFT_INIT_EDGE_CAP; + qsort(o->edges, (size_t)kept, sizeof(o->edges[0]), swift_init_edge_cmp); + + edges = NULL; + count = 0; + o->init_usages = 0; + if (cbm_store_find_edges_by_type(s, project, "USAGE", &edges, &count) == CBM_STORE_OK) { + for (int i = 0; i < count; i++) { + cbm_node_t tgt = {0}; + if (cbm_store_find_node_by_id(s, edges[i].target_id, &tgt) == CBM_STORE_OK && + tgt.name && (strcmp(tgt.name, "init") == 0 || strcmp(tgt.name, "deinit") == 0)) { + o->init_usages++; + } + cbm_node_free_fields(&tgt); + } + if (edges) { + cbm_store_free_edges(edges, count); + } + } +} + +/* `pad` filler files push the run past MIN_FILES_FOR_PARALLEL so that, with + * `parallel`, it goes through pass_parallel.c rather than pass_calls.c. */ +static void swift_init_case(bool parallel, int pad, SwiftInitObservation *o) { + memset(o, 0, sizeof(*o)); + o->run_rc = -1; + o->calls = -1; + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_swift_init_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + return; + } + char path[512]; + bool written = true; + const char *names[] = {"Package.swift", "Sources/Kit/Inits.swift", "Sources/Kit/Base.swift", + "Sources/App/Sub.swift"}; + const char *sources[] = {SWIFT_INIT_PACKAGE_SRC, SWIFT_INIT_INITS_SRC, SWIFT_INIT_BASE_SRC, + SWIFT_INIT_SUB_SRC}; + for (int i = 0; i < 4 && written; i++) { + snprintf(path, sizeof(path), "%s/%s", tmp, names[i]); + written = th_write_file(path, sources[i]) == 0; + } + for (int i = 0; i < pad && written; i++) { + char source[96]; + snprintf(path, sizeof(path), "%s/Pad%02d.swift", tmp, i); + snprintf(source, sizeof(source), "func pad%02d() -> Int { %d }\n", i, i); + written = th_write_file(path, source) == 0; + } + if (written) { + XlangEnvSave save = xlang_env_select(parallel); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/swift_init.db", tmp); + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + if (pipeline) { + o->run_rc = cbm_pipeline_run(pipeline); + const char *project = cbm_pipeline_project_name(pipeline); + cbm_store_t *store = cbm_store_open_path(db_path); + if (store && project) { + swift_init_observe(store, project, o); + } + if (store) { + cbm_store_close(store); + } + cbm_pipeline_free(pipeline); + } + xlang_env_restore(&save); + } + th_rmtree(tmp); +} + +/* The exact CALLS set. Every initializer's own calls come from that + * initializer (the repro's file-level collapse is gone); Sub.init reaches + * Base.init in the other target and `Box.init(raw:)` reaches Box.init. + * UIViewController's and UIAlertAction's initializers, Notification's own and + * the implicit `.init(...)` are outside the project and form no edge -- in + * particular none to a project `init` that merely shares the name. Holder's + * convenience `self.init` is not a self-loop. */ +static int swift_init_assert_exact(const SwiftInitObservation *o) { + static const char *const expected[] = { + "Sources.App.Sub.Sub.init -> Sources.Kit.Base.Base.init", + "Sources.App.Sub.explicit -> Sources.Kit.Inits.Box.init", + "Sources.Kit.Inits.Box.init -> Sources.Kit.Inits.helper", + "Sources.Kit.Inits.Box.init -> Sources.Kit.Inits.other", + "Sources.Kit.Inits.Box.read -> Sources.Kit.Inits.other", + "Sources.Kit.Inits.Holder.init -> Sources.Kit.Inits.helper", + "Sources.Kit.Inits.Holder.init -> Sources.Kit.Inits.other", + "Sources.Kit.Inits.Mode.init -> Sources.Kit.Inits.helper", + }; + enum { EXPECTED_CALLS = sizeof(expected) / sizeof(expected[0]) }; + ASSERT_EQ(o->run_rc, 0); + ASSERT_EQ(o->init_usages, 0); + ASSERT_EQ(o->calls, (int)EXPECTED_CALLS); + for (int i = 0; i < (int)EXPECTED_CALLS; i++) { + ASSERT_STR_EQ(o->edges[i], expected[i]); + } + return 0; +} + +TEST(pipeline_swift_init_exact_calls_sequential_issue2379) { + SwiftInitObservation o; + swift_init_case(false, 0, &o); + if (swift_init_assert_exact(&o) != 0) { + return 1; + } + PASS(); +} + +/* Same input through pass_parallel.c: 55 fillers exceed MIN_FILES_FOR_PARALLEL. */ +TEST(pipeline_swift_init_exact_calls_parallel_issue2379) { + SwiftInitObservation o; + swift_init_case(true, 55, &o); + if (swift_init_assert_exact(&o) != 0) { + return 1; + } + PASS(); +} + TEST(pipeline_python_cross_module_call) { /* Port of TestPythonCrossModuleCallViaImport */ const char *files[] = {"utils.py", "main.py"}; @@ -27638,6 +27885,8 @@ SUITE(pipeline) { RUN_TEST(pipeline_imports_multi_symbol_edges); RUN_TEST(pipeline_go_cross_package_call); RUN_TEST(pipeline_swift_cross_package_import); + RUN_TEST(pipeline_swift_init_exact_calls_sequential_issue2379); + RUN_TEST(pipeline_swift_init_exact_calls_parallel_issue2379); RUN_TEST(pipeline_python_cross_module_call); RUN_TEST(pipeline_cross_language_same_name_does_not_share_calls_issue725); RUN_TEST(pipeline_go_type_classification); From f683b856672c158bee68f12d532a51a382812ebd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 18:34:34 +0000 Subject: [PATCH 14/27] build(deps): bump actions/download-artifact from 8.0.1 to 8.0.2 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 8.0.1 to 8.0.2. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c...9000827ccba6bdab643e8b6fd33ac0654aef8333) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: 8.0.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/_build.yml | 2 +- .github/workflows/_smoke.yml | 6 +++--- .github/workflows/_soak.yml | 8 ++++---- .github/workflows/_test.yml | 4 ++-- .github/workflows/release.yml | 6 +++--- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/_build.yml b/.github/workflows/_build.yml index 152cca7376..dac1dde67c 100644 --- a/.github/workflows/_build.yml +++ b/.github/workflows/_build.yml @@ -317,7 +317,7 @@ jobs: persist-credentials: false - name: Download all native candidate pairs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: release-candidates-* path: ${{ runner.temp }}/release-candidate-artifacts diff --git a/.github/workflows/_smoke.yml b/.github/workflows/_smoke.yml index ed1022b718..932ee858d5 100644 --- a/.github/workflows/_smoke.yml +++ b/.github/workflows/_smoke.yml @@ -103,7 +103,7 @@ jobs: go-version-file: pkg/go/go.mod cache: false - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: binaries-${{ matrix.goos }}-${{ matrix.goarch }} @@ -203,7 +203,7 @@ jobs: zip coreutils - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: binaries-windows-${{ matrix.arch }} @@ -303,7 +303,7 @@ jobs: go-version-file: pkg/go/go.mod cache: false - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: binaries-linux-${{ matrix.arch }}-portable diff --git a/.github/workflows/_soak.yml b/.github/workflows/_soak.yml index 39f9e76b1d..4eb0ac44a1 100644 --- a/.github/workflows/_soak.yml +++ b/.github/workflows/_soak.yml @@ -73,7 +73,7 @@ jobs: run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=${{ matrix.cc }} CXX=${{ matrix.cxx }} - name: Download selected release container if: ${{ inputs.use_release_artifacts }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: binaries-${{ matrix.goos }}-${{ matrix.goarch }} path: ${{ runner.temp }}/cbm-selected-container @@ -147,7 +147,7 @@ jobs: run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++ - name: Download selected release container if: ${{ inputs.use_release_artifacts }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: binaries-windows-amd64 path: ${{ runner.temp }}/cbm-selected-container @@ -225,7 +225,7 @@ jobs: run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++ - name: Download selected release container if: ${{ inputs.use_release_artifacts }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: binaries-windows-arm64 path: ${{ runner.temp }}/cbm-selected-container @@ -287,7 +287,7 @@ jobs: timeout-minutes: 10 run: APT_RETRIES=3; [ -f /etc/apt/apt-mirrors.txt ] && APT_RETRIES=0; sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 --error-on=any update && sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 install -y python3 git - name: Download selected portable release container - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: binaries-linux-${{ matrix.arch }}-portable path: ${{ runner.temp }}/cbm-selected-container diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 501a82b205..2f89e79e6d 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -231,7 +231,7 @@ jobs: # producer, so arm64, macOS and Windows code under #ifdef has static # evidence only and those legs run every test. if: ${{ inputs.test_selection != '' && runner.os == 'Linux' && runner.arch == 'X64' }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: ${{ inputs.test_selection }} path: ${{ runner.temp }}/test-selection @@ -720,7 +720,7 @@ jobs: with: path: repo - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: shard-manifest-* path: manifests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e4926260d5..c70330c7e5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -185,7 +185,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Download only selected release containers - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: binaries-* merge-multiple: true @@ -194,7 +194,7 @@ jobs: # download above. Preserve only the three reviewable manifests that bind # candidate bytes, VT results, and the tuple-local decision. - name: Download candidate selection evidence separately - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: release-selection-evidence path: selection-evidence @@ -617,7 +617,7 @@ jobs: # the release-draft job as a same-run artifact (not a draft-release # download — that would need contents beyond read plus gh). - name: Fetch release checksums - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: release-checksums From bf30a1aaf01d01f750fdfa695baec34ed977dd53 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 1 Nov 2026 23:48:31 +0100 Subject: [PATCH 15/27] fix(daemon): replay the session within the startup budget The recovery path replays the session context and UI settings on the replacement daemon. It waited on the bootstrap connect budget, which only bounds a probe (1 s in production), while main.c gives the first set_context and set_ui_config the startup budget since eb294298e. A replacement daemon whose UI save runs longer than a probe would end an otherwise good recovery. frontend_configure_client now waits on startup_timeout_ms. A test seam records the budget the replay used, and the recovery fixture gives the bootstrap distinct connect and startup budgets and exits 75 when the replay waited on the wrong one. With the old budget exactly the three recovery tests fail with exit 75; with the fix daemon_frontend is 17/17. Signed-off-by: Martin Vogel --- src/daemon/frontend.c | 14 +++++++++++++- src/daemon/frontend.h | 4 ++++ tests/test_daemon_frontend.c | 15 ++++++++++++++- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/daemon/frontend.c b/src/daemon/frontend.c index 123ec00e47..e76c920b6c 100644 --- a/src/daemon/frontend.c +++ b/src/daemon/frontend.c @@ -120,6 +120,7 @@ static atomic_uint_fast64_t frontend_test_monitor_observation_count = ATOMIC_VAR static atomic_uint_fast64_t frontend_test_worker_observation_count = ATOMIC_VAR_INIT(0); static atomic_uint_fast64_t frontend_test_worker_idle_count = ATOMIC_VAR_INIT(0); static atomic_uint_fast64_t frontend_test_routed_cancellation_count = ATOMIC_VAR_INIT(0); +static atomic_uint_fast32_t frontend_test_replay_budget_ms = ATOMIC_VAR_INIT(0); void cbm_daemon_frontend_test_observer_reset(bool hold_monitor) { atomic_store_explicit(&frontend_test_monitor_observation_count, 0, memory_order_release); @@ -152,6 +153,10 @@ uint64_t cbm_daemon_frontend_test_worker_idle_cycles(void) { uint64_t cbm_daemon_frontend_test_routed_cancellations(void) { return atomic_load_explicit(&frontend_test_routed_cancellation_count, memory_order_acquire); } + +uint32_t cbm_daemon_frontend_test_replay_timeout_ms(void) { + return (uint32_t)atomic_load_explicit(&frontend_test_replay_budget_ms, memory_order_acquire); +} #endif static cbm_version_cohort_maintenance_presence_t frontend_observe_maintenance( @@ -528,7 +533,14 @@ static cbm_daemon_runtime_cancel_result_t frontend_cancel_active(frontend_state_ static bool frontend_configure_client(cbm_daemon_runtime_client_t *client, const cbm_daemon_frontend_session_config_t *session) { - uint32_t timeout_ms = session->bootstrap.connect_timeout_ms; + /* The replay restores what startup set, so it waits within the startup + * budget like the first set_context and set_ui_config in main.c. The + * connect budget only bounds a probe (1 s in production): a replacement + * daemon whose UI save runs longer would end an otherwise good recovery. */ + uint32_t timeout_ms = session->bootstrap.startup_timeout_ms; +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS + atomic_store_explicit(&frontend_test_replay_budget_ms, timeout_ms, memory_order_release); +#endif if (cbm_daemon_application_client_set_context( client, session->session_root, session->allowed_root, session->tool_profile, NULL, NULL, timeout_ms) != CBM_DAEMON_RUNTIME_APPLICATION_OK) { diff --git a/src/daemon/frontend.h b/src/daemon/frontend.h index 8f59ec28b5..8ee33d2765 100644 --- a/src/daemon/frontend.h +++ b/src/daemon/frontend.h @@ -79,6 +79,10 @@ uint64_t cbm_daemon_frontend_test_worker_idle_cycles(void); * Tests snapshot this monotonic * count before sending a notification. */ uint64_t cbm_daemon_frontend_test_routed_cancellations(void); +/* Budget in ms that the most recent session replay gave its context and UI + * settings requests. Recovery tests use distinct connect and startup budgets + * and compare. */ +uint32_t cbm_daemon_frontend_test_replay_timeout_ms(void); #endif /* Takes ownership of client and borrows cohort_manager for the complete call. diff --git a/tests/test_daemon_frontend.c b/tests/test_daemon_frontend.c index e4ed5e135b..aa49bf4c72 100644 --- a/tests/test_daemon_frontend.c +++ b/tests/test_daemon_frontend.c @@ -250,6 +250,11 @@ enum { * no-progress EOF drain so shutdown deterministically overlaps recovery. */ FRONTEND_RECOVERY_EOF_RELEASE_MS = 16000, + /* Startup budget of the recovery bootstrap. It differs from the connect + * budget (FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS) so the recovery tests can + * tell which one the session replay waited on, and it is the larger one, + * as in production (1 s connect probe, 30 s startup). */ + FRONTEND_RECOVERY_TEST_STARTUP_TIMEOUT_MS = 45000, }; typedef enum { @@ -818,7 +823,7 @@ static int frontend_recovery_child_run(const char *parent, frontend_recovery_mod .identity = &fixture.identity, .executable_path = "unused", .connect_timeout_ms = FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS, - .startup_timeout_ms = FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS, + .startup_timeout_ms = FRONTEND_RECOVERY_TEST_STARTUP_TIMEOUT_MS, }, .session_root = parent, .tool_profile = CBM_MCP_TOOL_PROFILE_ALL, @@ -851,6 +856,14 @@ static int frontend_recovery_child_run(const char *parent, frontend_recovery_mod bool input_closed = input && fclose(input) == 0; bool output_closed = output && fclose(output) == 0; bool fixture_closed = frontend_eof_fixture_finish(&fixture); +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS + /* The replay restores what startup set within the startup budget. On the + * connect budget (a probe bound) a replacement daemon that answers slower + * than a probe would end an otherwise good recovery. Every mode replays. */ + if (cbm_daemon_frontend_test_replay_timeout_ms() != FRONTEND_RECOVERY_TEST_STARTUP_TIMEOUT_MS) { + return 75; + } +#endif return replacement_ready && request_ready && result == 0 && response_exact && requests == expected_requests && writer_ok && input_closed && output_closed && fixture_closed From 767ac3781c1f672d138ff63b4a4bb5dcc504c5bf Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 2 Nov 2026 00:16:13 +0100 Subject: [PATCH 16/27] test(index): pin the duration clock in the quiet-timeout watchdog test index_supervisor_quiet_timeout_remains_hang_with_duration_enabled ran the duration watchdog on the real clock with a 60 s limit next to a 1 s quiet timeout. On a starved host the duration limit can be crossed before the quiet timeout fires, and the verdict then turns from HANG into a resource kill, so the test depended on how fast the machine was. The resource clock is now pinned at the request origin through the existing clock seam. The quiet timeout runs on the subprocess clock, so it is the only way the hung worker can end, however slowly the host runs. The limit is set equal to the quiet timeout on purpose: more real time than the limit always passes and it still cannot fire. A new assertion checks that the duration check read the pinned clock, so HANG wins against an armed watchdog. RED (real clock, limit crossed first): FAIL ASSERT(remained_hang) with index.supervisor.resource_limit resource=duration_ms. GREEN with the pin. Signed-off-by: Martin Vogel --- tests/test_index_supervisor.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/test_index_supervisor.c b/tests/test_index_supervisor.c index df55407da5..bda5032384 100644 --- a/tests/test_index_supervisor.c +++ b/tests/test_index_supervisor.c @@ -1230,11 +1230,22 @@ TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled) { const char *saved_timeout = getenv("CBM_INDEX_WORKER_TIMEOUT_S"); char *saved_timeout_copy = saved_timeout ? cbm_strdup(saved_timeout) : NULL; (void)cbm_setenv("CBM_INDEX_WORKER_TIMEOUT_S", "1", 1); - cbm_index_supervisor_reset_resource_hooks_for_testing(); - cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 60000); + /* The duration watchdog stays armed, but its clock is pinned at the request + * origin, so the quiet timeout (the subprocess's own clock) is the only way + * this hung worker can end, however slowly the host runs. With the real + * clock a starved run could cross the duration limit first and turn the + * HANG into a resource kill. The limit equals the quiet timeout on purpose: + * more real time than the limit always passes, and it still cannot fire. */ + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_EMPTY, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_charged_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, - &policy, false, NULL, NULL, 0, &handle); + &policy, false, NULL, NULL, 100, &handle); const cbm_index_worker_result_t *result = NULL; bool terminal = handle && index_supervisor_test_poll_terminal( handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); @@ -1246,11 +1257,15 @@ TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled) { } else { index_supervisor_test_cleanup_handle(handle); } + cbm_index_supervisor_reset_resource_hooks_for_testing(); index_supervisor_test_restore_env("CBM_INDEX_WORKER_TIMEOUT_S", saved_timeout_copy); ASSERT_EQ(start_rc, 0); ASSERT_TRUE(terminal); ASSERT_TRUE(remained_hang); + /* The duration check really ran while polling (it read the pinned clock), + * so HANG won against a live watchdog, not against a disabled one. */ + ASSERT_TRUE(fake.clock_calls > 0); PASS(); } From 33ae8f382e20fa39f6854f85bdfdc5fdf9b1ec88 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 2 Nov 2026 00:16:48 +0100 Subject: [PATCH 17/27] test(subprocess): prove descendant memory is measured on every platform subprocess_tree_rss_measures_contained_descendants asserted only that the tree total was above zero, which the root alone satisfies, and it was skipped on Windows although its skip text said it ran there. The test runner gains two roles: __cbm_rss_tree_root starts one __cbm_rss_tree_hold descendant (fork on POSIX, CreateProcessW into the same Job Object on Windows) that touches a controlled allocation and publishes its pid by an atomic rename. The test measures two otherwise identical trees, one whose descendant holds nothing and one whose descendant holds 128 MiB, and requires the second total to be at least 96 MiB higher. Only a probe that measures past the root can pass. The test now runs natively on Windows. The waits have no deadline: they end when the descendant is ready or the tree stops running, and the root exits when its descendant dies. Binding check: with the macOS probe reduced to the root only, the test fails (the loaded tree is not 96 MiB above the idle one). GREEN on the real probe (idle 43 MiB, loaded 187 MiB under ASan). Signed-off-by: Martin Vogel --- tests/test_main.c | 97 ++++++++++++++++++++++++++ tests/test_subprocess.c | 146 +++++++++++++++++++++++++++++++--------- 2 files changed, 210 insertions(+), 33 deletions(-) diff --git a/tests/test_main.c b/tests/test_main.c index e0526cf858..23b2621ee7 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -44,6 +44,7 @@ int tf_deselected_count = 0; #include #include "foundation/win_utf8.h" /* cbm_module_path_utf8 — the runner's own image */ #else +#include /* waitpid: the process-tree probe root */ #include #ifdef __APPLE__ #include @@ -1968,6 +1969,100 @@ static int tf_role_env_isolation(int argc, char **argv) { return 0; } +/* Process-tree memory probe coverage (test_subprocess.c). The root starts ONE + * descendant that charges a controlled amount of memory, so a tree total can + * be checked against an allocation only a descendant holds. The descendant + * publishes its pid by an atomic rename once every page is touched, then + * blocks until it is killed. The root exits when its descendant dies, so a + * failed setup ends the tree instead of leaving the test waiting. */ +static int tf_rss_tree_hold(const char *ready_path, const char *mib_text) { + char *end = NULL; + long mib = strtol(mib_text, &end, 10); + if (!end || *end != '\0' || mib < 0 || mib > 1024) { + return 81; + } + size_t bytes = (size_t)mib * 1024U * 1024U; + unsigned char *block = bytes > 0 ? malloc(bytes) : NULL; + if (bytes > 0 && !block) { + return 82; + } + if (block) { + memset(block, 0xA5, bytes); + } + char staged[1024]; + int length = snprintf(staged, sizeof(staged), "%s.tmp", ready_path); + FILE *ready = length > 0 && (size_t)length < sizeof(staged) ? cbm_fopen(staged, "wb") : NULL; + if (!ready) { + return 83; + } +#ifdef _WIN32 + (void)fprintf(ready, "%lu\n", (unsigned long)GetCurrentProcessId()); +#else + (void)fprintf(ready, "%ld\n", (long)getpid()); +#endif + if (fclose(ready) != 0 || rename(staged, ready_path) != 0) { + return 84; + } + volatile unsigned char keep = block ? block[bytes - 1] : 0; + (void)keep; + for (;;) { +#ifdef _WIN32 + Sleep(INFINITE); +#else + (void)pause(); +#endif + } +} + +static int tf_role_rss_tree_hold(int argc, char **argv) { + return argc == 4 ? tf_rss_tree_hold(argv[2], argv[3]) : -1; +} + +static int tf_role_rss_tree_root(int argc, char **argv) { + if (argc != 4) { + return -1; + } +#ifdef _WIN32 + char *self = cbm_module_path_utf8(); + const char *child_argv[] = {self, "__cbm_rss_tree_hold", argv[2], argv[3], NULL}; + char command_line[8192]; + bool built = self && cbm_build_win_cmdline(command_line, sizeof(command_line), child_argv); + wchar_t *wide_self = built ? cbm_utf8_to_wide(self) : NULL; + wchar_t *wide_command = built ? cbm_utf8_to_wide(command_line) : NULL; + STARTUPINFOW startup; + ZeroMemory(&startup, sizeof(startup)); + startup.cb = sizeof(startup); + PROCESS_INFORMATION child; + ZeroMemory(&child, sizeof(child)); + /* The child joins this process's Job Object by default, which is the tree + * the probe under test enumerates. */ + BOOL started = wide_self && wide_command && + CreateProcessW(wide_self, wide_command, NULL, NULL, FALSE, 0, NULL, NULL, + &startup, &child); + free(wide_self); + free(wide_command); + free(self); + if (!started) { + return 85; + } + CloseHandle(child.hThread); + (void)WaitForSingleObject(child.hProcess, INFINITE); + CloseHandle(child.hProcess); +#else + pid_t child = fork(); + if (child < 0) { + return 85; + } + if (child == 0) { + _exit(tf_rss_tree_hold(argv[2], argv[3])); + } + int status = 0; + while (waitpid(child, &status, 0) < 0 && errno == EINTR) { + } +#endif + return 86; /* the descendant ended: end the tree */ +} + /* Installation tests use this executable as a structurally real candidate. * Mirror the production binary's minimal verification contract. */ static int tf_role_version(int argc, char **argv) { @@ -2060,6 +2155,8 @@ typedef struct { static const tf_role_t tf_roles[] = { TF_ROLE(TF_PHASE_RAW, "__cbm_env_values", tf_role_env_values), TF_ROLE(TF_PHASE_RAW, "__cbm_env_isolation", tf_role_env_isolation), + TF_ROLE(TF_PHASE_RAW, "__cbm_rss_tree_root", tf_role_rss_tree_root), + TF_ROLE(TF_PHASE_RAW, "__cbm_rss_tree_hold", tf_role_rss_tree_hold), TF_ROLE(TF_PHASE_GIT_CLEAN, "__cbm_subprocess_stdin_probe", tf_role_subprocess_stdin_probe), TF_ROLE(TF_PHASE_GIT_CLEAN, "__cbm_subprocess_stdout_probe", tf_role_subprocess_stdout_probe), TF_ROLE(TF_PHASE_GIT_CLEAN, "__cbm_git_facts_diff_probe", tf_role_git_facts_diff_probe), diff --git a/tests/test_subprocess.c b/tests/test_subprocess.c index 385293e085..1c5039e87a 100644 --- a/tests/test_subprocess.c +++ b/tests/test_subprocess.c @@ -807,43 +807,123 @@ TEST(subprocess_spawn_returns_while_child_is_running) { #endif } -TEST(subprocess_tree_rss_measures_contained_descendants) { -#ifdef _WIN32 - SKIP_PLATFORM("native Job Object process-tree RSS runs on Windows CI"); -#else - char pid_path[64]; - ASSERT_TRUE(make_tree_pid_path(pid_path)); - cbm_subprocess_t *process = NULL; - ASSERT_EQ(spawn_ignoring_tree(pid_path, 0, 100, &process), 0); - pid_t parent_pid = 0; - pid_t grandchild_pid = 0; - bool ready = wait_for_tree_pids(pid_path, process, &parent_pid, &grandchild_pid, 3000); - uint64_t rss_bytes = 0; - cbm_proc_tree_rss_status_t rss_status = - ready ? cbm_subprocess_tree_rss_bytes(process, &rss_bytes) : CBM_PROC_TREE_RSS_ERROR; - bool cancel_accepted = cbm_subprocess_request_cancel(process); +/* ── Process-tree memory probe (every platform) ───────────────────────────── + * The test runner's __cbm_rss_tree_root role starts one __cbm_rss_tree_hold + * descendant that touches a controlled allocation and then publishes its pid. + * Waits below have no deadline: they end when the descendant is ready or the + * tree stops running, so the outcome, not a timer, decides the verdict. */ + +#define RSS_TREE_MIB UINT64_C(1048576) + +typedef struct { + bool spawned; + bool ready; + cbm_proc_tree_rss_status_t status; + uint64_t rss; + bool quiesced; + cbm_proc_tree_rss_status_t final_status; + uint64_t final_rss; +} rss_tree_sample_t; + +static void rss_tree_pause(void) { + cbm_usleep(10000); /* poll interval only; nothing here is a deadline */ +} + +static bool rss_tree_wait_ready(const char *ready_path, cbm_subprocess_t *process) { + for (;;) { + FILE *ready = cbm_fopen(ready_path, "rb"); + if (ready) { + long pid = 0; + int fields = fscanf(ready, "%ld", &pid); + (void)fclose(ready); + if (fields == 1 && pid > 0) { + return true; + } + } + cbm_proc_result_t ignored; + if (cbm_subprocess_poll(process, &ignored) != CBM_PROC_POLL_RUNNING) { + return false; + } + rss_tree_pause(); + } +} + +static bool rss_tree_finish(cbm_subprocess_t *process, rss_tree_sample_t *sample) { + (void)cbm_subprocess_request_cancel(process); cbm_proc_result_t result; - bool terminal = poll_until_terminal(process, 5000, &result); - uint64_t final_rss = UINT64_MAX; - cbm_proc_tree_rss_status_t final_rss_status = - terminal ? cbm_subprocess_tree_rss_bytes(process, &final_rss) : CBM_PROC_TREE_RSS_ERROR; - if (terminal) { - cbm_subprocess_destroy(process); - } else { - force_probe_cleanup(parent_pid, grandchild_pid); + for (;;) { + cbm_proc_poll_t state = cbm_subprocess_poll(process, &result); + if (state == CBM_PROC_POLL_TERMINAL) { + break; + } + if (state == CBM_PROC_POLL_ERROR) { + return false; + } + rss_tree_pause(); } - (void)unlink(pid_path); + sample->final_status = cbm_subprocess_tree_rss_bytes(process, &sample->final_rss); + cbm_subprocess_destroy(process); + return result.tree_quiesced; +} + +/* Measure one tree whose descendant holds `mib` MiB. */ +static rss_tree_sample_t rss_tree_sample(int mib) { + rss_tree_sample_t sample = {.status = CBM_PROC_TREE_RSS_ERROR, + .final_status = CBM_PROC_TREE_RSS_ERROR, + .final_rss = UINT64_MAX}; + const char *made = th_mktempdir("cbm_rss_tree"); + char dir[256]; + char ready_path[300]; + char mib_text[16]; + if (!made || snprintf(dir, sizeof(dir), "%s", made) >= (int)sizeof(dir) || + snprintf(ready_path, sizeof(ready_path), "%s/ready", dir) >= (int)sizeof(ready_path)) { + return sample; + } + (void)snprintf(mib_text, sizeof(mib_text), "%d", mib); + const char *argv[] = {cbm_test_program_path, "__cbm_rss_tree_root", ready_path, mib_text, + NULL}; + cbm_proc_opts_t opts = {0}; + opts.bin = cbm_test_program_path; + opts.argv = argv; + opts.cancel_grace_ms = 100; + cbm_subprocess_t *process = NULL; + sample.spawned = cbm_subprocess_spawn(&opts, &process) == 0; + if (sample.spawned) { + sample.ready = rss_tree_wait_ready(ready_path, process); + if (sample.ready) { + sample.status = cbm_subprocess_tree_rss_bytes(process, &sample.rss); + } + sample.quiesced = rss_tree_finish(process, &sample); + } + th_cleanup(dir); + return sample; +} - ASSERT_TRUE(ready); - ASSERT_EQ(rss_status, CBM_PROC_TREE_RSS_OK); - ASSERT_TRUE(rss_bytes > 0); - ASSERT_TRUE(cancel_accepted); - ASSERT_TRUE(terminal); - ASSERT_TRUE(result.tree_quiesced); - ASSERT_EQ(final_rss_status, CBM_PROC_TREE_RSS_EMPTY); - ASSERT_EQ(final_rss, 0); +/* Only the DESCENDANT allocates, so the difference between two otherwise + * identical trees is memory the probe can only have counted by measuring past + * the root. 128 MiB touched against a 96 MiB floor leaves room for allocator + * and page-rounding noise and none for a root-only total. */ +TEST(subprocess_tree_rss_measures_contained_descendants) { + rss_tree_sample_t idle = rss_tree_sample(0); + rss_tree_sample_t loaded = rss_tree_sample(128); + if (loaded.status == CBM_PROC_TREE_RSS_OK && idle.status == CBM_PROC_TREE_RSS_OK) { + printf(" tree rss: idle=%llu MiB loaded=%llu MiB\n", + (unsigned long long)(idle.rss / RSS_TREE_MIB), + (unsigned long long)(loaded.rss / RSS_TREE_MIB)); + } + ASSERT_TRUE(idle.spawned); + ASSERT_TRUE(idle.ready); + ASSERT_EQ(idle.status, CBM_PROC_TREE_RSS_OK); + ASSERT_TRUE(idle.rss > 0); + ASSERT_TRUE(loaded.spawned); + ASSERT_TRUE(loaded.ready); + ASSERT_EQ(loaded.status, CBM_PROC_TREE_RSS_OK); + ASSERT_TRUE(loaded.rss >= idle.rss + 96 * RSS_TREE_MIB); + ASSERT_TRUE(idle.quiesced); + ASSERT_TRUE(loaded.quiesced); + ASSERT_EQ(loaded.final_status, CBM_PROC_TREE_RSS_EMPTY); + ASSERT_EQ(loaded.final_rss, 0); PASS(); -#endif } TEST(subprocess_natural_completion_is_cached_across_polls) { From c78c077a507100201a067b1ed49588df4d06f1f7 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 2 Nov 2026 00:27:33 +0100 Subject: [PATCH 18/27] fix(subprocess): fail the tree probe on a live member it cannot measure cbm_subprocess_tree_rss_bytes skipped any non-root process it could not measure and still returned OK with a smaller total, so a descendant the platform refused to measure silently dropped out of the RSS watchdog. Now a member is skipped only when the platform says it no longer exists: ESRCH from proc_pid_rusage on macOS, ENOENT or ESRCH for /proc//stat on Linux, ERROR_INVALID_PARAMETER from OpenProcess (or an exit code from a handle whose memory query failed) on Windows. Any other failure fails the probe, which the supervisor already turns into resource_probe_failed after three consecutive errors. The root stays strict as before. On macOS the members now come from proc_listpids(PROC_PGRP_ONLY): the old all-process scan read every process with proc_pidinfo, which answers EPERM for every process of another user (178 of 6438 on this machine), so it could not tell a foreign process from an unreadable member and fail-closed semantics would have failed every probe there. Linux keeps its /proc scan; stat files are world-readable on a default mount, and a hidepid=1 mount now fails closed, which the doc states. A test seam makes every non-root measurement fail with one platform error. RED (old skip-any-failure decision): the live-member test returns OK where ERROR is required. GREEN with the fix; the exited-member test passes in both, so verified exit races are still ignored. Signed-off-by: Martin Vogel --- docs/INDEX_RESOURCE_LIMITS.md | 6 +- src/foundation/subprocess.c | 208 +++++++++++++++++++++++++--------- src/foundation/subprocess.h | 9 +- tests/test_subprocess.c | 47 +++++++- 4 files changed, 208 insertions(+), 62 deletions(-) diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index 1525993606..11fcf4a3d4 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -60,7 +60,11 @@ quarantine a source file. If RSS is enabled and three consecutive probes cannot obtain any trustworthy tree measurement while the root worker is still running, CBM fails closed with -`code=resource_probe_failed`. +`code=resource_probe_failed`. A probe is untrustworthy as soon as one member of +the tree cannot be measured: only a process that exits between the listing and +its measurement is skipped, because a total without a live member would +under-report. On Linux the probe reads every `/proc//stat` to find the +tree's members, so a `/proc` mounted with `hidepid=1` makes it fail closed. ## Counting and failure semantics diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 519a554e23..bce08c6bff 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -476,9 +476,53 @@ uint64_t cbm_subprocess_rss_sum_for_testing(const uint64_t *values, size_t count } return total; } + +static atomic_int g_tree_rss_member_fault; + +void cbm_subprocess_tree_rss_fault_members_for_testing(int platform_error) { + atomic_store_explicit(&g_tree_rss_member_fault, platform_error, memory_order_release); +} + +/* Test builds can make every NON-root member's measurement fail with one + * platform error (errno on POSIX, GetLastError() on Windows); 0 = real call. + * Production builds compile the check out entirely. */ +static int cbm_tree_rss_member_fault(bool root) { + return root ? 0 : atomic_load_explicit(&g_tree_rss_member_fault, memory_order_acquire); +} #endif +/* A member that cannot be measured is skipped only when the platform says it + * no longer exists (it exited between the listing and the measurement, and + * holds nothing). Any other failure is a live member whose memory a total + * would silently leave out, so the probe fails instead -- a watchdog that + * under-reports is one that quietly stopped watching. The root is strict + * either way. */ #ifdef _WIN32 +static HANDLE cbm_tree_rss_open_member(DWORD pid, bool root, DWORD *error_out) { +#ifdef CBM_ENABLE_TEST_SEAMS + int fault = cbm_tree_rss_member_fault(root); + if (fault != 0) { + *error_out = (DWORD)fault; + return NULL; + } +#else + (void)root; +#endif + /* GetProcessMemoryInfo documents PROCESS_QUERY_INFORMATION | + * PROCESS_VM_READ. PROCESS_QUERY_LIMITED_INFORMATION is a lesser + * alternative on newer Windows; the Job Object membership list + * already closes most of the pid-reuse window between listing and + * opening. */ + HANDLE member = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid); + *error_out = member ? ERROR_SUCCESS : GetLastError(); + return member; +} + +static bool cbm_tree_rss_member_exited(HANDLE member) { + DWORD exit_code = 0; + return GetExitCodeProcess(member, &exit_code) && exit_code != STILL_ACTIVE; +} + static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, uint64_t *rss_bytes) { DWORD capacity = 32; @@ -510,18 +554,15 @@ static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subproces uint64_t total = 0; DWORD measured = 0; - bool root_failed = false; - for (DWORD index = 0; index < processes->NumberOfProcessIdsInList; index++) { + bool failed = false; + for (DWORD index = 0; index < processes->NumberOfProcessIdsInList && !failed; index++) { DWORD pid = (DWORD)processes->ProcessIdList[index]; - /* GetProcessMemoryInfo documents PROCESS_QUERY_INFORMATION | - * PROCESS_VM_READ. PROCESS_QUERY_LIMITED_INFORMATION is a lesser - * alternative on newer Windows; the Job Object membership list - * already closes most of the pid-reuse window between listing and - * opening. A member we cannot open is a probe failure when it is - * the root. */ - HANDLE member = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid); + bool root = pid == process->process_id; + DWORD open_error = ERROR_SUCCESS; + HANDLE member = cbm_tree_rss_open_member(pid, root, &open_error); if (!member) { - root_failed = root_failed || pid == process->process_id; + /* ERROR_INVALID_PARAMETER: no process has this id any more. */ + failed = root || open_error != ERROR_INVALID_PARAMETER; continue; } PROCESS_MEMORY_COUNTERS memory; @@ -529,14 +570,14 @@ static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subproces if (GetProcessMemoryInfo(member, &memory, sizeof(memory))) { cbm_rss_add_saturated(&total, (uint64_t)memory.WorkingSetSize); measured++; - } else if (pid == process->process_id) { - root_failed = true; + } else { + failed = root || !cbm_tree_rss_member_exited(member); } CloseHandle(member); } DWORD listed = processes->NumberOfProcessIdsInList; cbm_free(CBM_MEM_CLASS_OTHER, processes); - if (root_failed || (measured == 0 && listed > 0)) { + if (failed || (measured == 0 && listed > 0)) { return CBM_PROC_TREE_RSS_ERROR; } if (measured == 0) { @@ -546,56 +587,73 @@ static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subproces return CBM_PROC_TREE_RSS_OK; } #elif defined(__APPLE__) -static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, - uint64_t *rss_bytes) { - int capacity = proc_listallpids(NULL, 0); - if (capacity <= 0 || capacity > INT_MAX / (int)sizeof(pid_t) - 64) { - return CBM_PROC_TREE_RSS_ERROR; - } - capacity += 64; +/* The members of the contained process group, as the kernel lists them. An + * all-process scan cannot decide membership: proc_pidinfo answers EPERM for + * every process of another user, so "not in our group" and "in our group but + * unreadable" would look the same. Returns the member count, or -1. */ +static int cbm_tree_rss_list_group(pid_t pgid, pid_t **pids_out) { + int capacity = 64; pid_t *pids = NULL; - int count = 0; for (;;) { pid_t *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, pids, (size_t)capacity * sizeof(*pids)); if (!grown) { cbm_free(CBM_MEM_CLASS_OTHER, pids); - return CBM_PROC_TREE_RSS_ERROR; + return -1; } pids = grown; - count = proc_listallpids(pids, capacity * (int)sizeof(*pids)); - if (count <= 0) { + errno = 0; + int bytes = + proc_listpids(PROC_PGRP_ONLY, (uint32_t)pgid, pids, capacity * (int)sizeof(*pids)); + if (bytes < 0 || (bytes == 0 && errno != 0)) { cbm_free(CBM_MEM_CLASS_OTHER, pids); - return CBM_PROC_TREE_RSS_ERROR; + return -1; } + int count = bytes / (int)sizeof(*pids); if (count < capacity) { - break; + *pids_out = pids; + return count; } if (capacity > INT_MAX / (int)sizeof(*pids) / 2) { cbm_free(CBM_MEM_CLASS_OTHER, pids); - return CBM_PROC_TREE_RSS_ERROR; + return -1; } capacity *= 2; } +} + +static int cbm_tree_rss_member_rusage(pid_t pid, bool root, struct rusage_info_v2 *usage) { +#ifdef CBM_ENABLE_TEST_SEAMS + int fault = cbm_tree_rss_member_fault(root); + if (fault != 0) { + errno = fault; + return -1; + } +#else + (void)root; +#endif + return proc_pid_rusage(pid, RUSAGE_INFO_V2, (rusage_info_t *)usage); +} + +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + pid_t *pids = NULL; + int count = cbm_tree_rss_list_group(process->pgid, &pids); + if (count < 0) { + return CBM_PROC_TREE_RSS_ERROR; + } uint64_t total = 0; int measured = 0; - bool root_failed = false; - for (int index = 0; index < count; index++) { - struct proc_bsdinfo info; + bool failed = false; + for (int index = 0; index < count && !failed; index++) { if (pids[index] <= 0) { continue; } - if (proc_pidinfo(pids[index], PROC_PIDTBSDINFO, 0, &info, sizeof(info)) != - (int)sizeof(info)) { - root_failed = root_failed || pids[index] == process->pid; - continue; - } - if ((pid_t)info.pbi_pgid != process->pgid) { - continue; - } + bool root = pids[index] == process->pid; struct rusage_info_v2 usage; - if (proc_pid_rusage(pids[index], RUSAGE_INFO_V2, (rusage_info_t *)&usage) != 0) { - root_failed = root_failed || pids[index] == process->pid; + if (cbm_tree_rss_member_rusage(pids[index], root, &usage) != 0) { + /* ESRCH: the member exited after the listing. */ + failed = root || errno != ESRCH; continue; } /* phys_footprint is what the OS charges, matching cbm_mem_charged(). @@ -606,7 +664,7 @@ static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subproces measured++; } cbm_free(CBM_MEM_CLASS_OTHER, pids); - if (root_failed) { + if (failed) { return CBM_PROC_TREE_RSS_ERROR; } if (measured == 0) { @@ -628,22 +686,61 @@ static bool cbm_proc_pid_name(const char *name) { return true; } -static bool cbm_linux_proc_stat(const char *pid_name, pid_t *group, int64_t *rss_pages) { +typedef enum { + CBM_LINUX_STAT_OK = 0, + CBM_LINUX_STAT_GONE, /* the process exited after the /proc listing */ + CBM_LINUX_STAT_UNREADABLE, /* a process whose stat could not be read or parsed */ +} cbm_linux_stat_t; + +/* /proc//stat is world-readable, so on a default /proc mount the only + * expected failure is the exit race: ENOENT at open, ESRCH at read. Anything + * else (EACCES under hidepid=1, a parse failure) leaves the process's group + * unknown, and the probe fails closed rather than guess it is not ours. */ +static cbm_linux_stat_t cbm_linux_stat_failure(int error) { + return error == ENOENT || error == ESRCH ? CBM_LINUX_STAT_GONE : CBM_LINUX_STAT_UNREADABLE; +} + +static cbm_linux_stat_t cbm_linux_proc_stat_line(const char *pid_name, bool root, char *line, + size_t line_size) { char path[64]; int length = snprintf(path, sizeof(path), "/proc/%s/stat", pid_name); if (length <= 0 || length >= (int)sizeof(path)) { - return false; + return CBM_LINUX_STAT_UNREADABLE; + } +#ifdef CBM_ENABLE_TEST_SEAMS + int fault = cbm_tree_rss_member_fault(root); + if (fault != 0) { + return cbm_linux_stat_failure(fault); } +#else + (void)root; +#endif + errno = 0; FILE *stat_file = cbm_fopen(path, "rb"); if (!stat_file) { - return false; + return cbm_linux_stat_failure(errno); } - char stat_line[4096]; - bool read = fgets(stat_line, sizeof(stat_line), stat_file) != NULL; + errno = 0; + bool read = fgets(line, (int)line_size, stat_file) != NULL; + int read_error = errno; (void)fclose(stat_file); - char *command_end = read ? strrchr(stat_line, ')') : NULL; + if (!read) { + return read_error == ESRCH ? CBM_LINUX_STAT_GONE : CBM_LINUX_STAT_UNREADABLE; + } + return CBM_LINUX_STAT_OK; +} + +static cbm_linux_stat_t cbm_linux_proc_stat(const char *pid_name, bool root, pid_t *group, + int64_t *rss_pages) { + char stat_line[4096]; + cbm_linux_stat_t status = + cbm_linux_proc_stat_line(pid_name, root, stat_line, sizeof(stat_line)); + if (status != CBM_LINUX_STAT_OK) { + return status; + } + char *command_end = strrchr(stat_line, ')'); if (!command_end || command_end[1] != ' ') { - return false; + return CBM_LINUX_STAT_UNREADABLE; } char *save = NULL; @@ -656,7 +753,7 @@ static bool cbm_linux_proc_stat(const char *pid_name, pid_t *group, int64_t *rss char *end = NULL; long long value = strtoll(token, &end, 10); if (!end || *end != '\0') { - return false; + return CBM_LINUX_STAT_UNREADABLE; } if (field == 5) { *group = (pid_t)value; @@ -670,7 +767,7 @@ static bool cbm_linux_proc_stat(const char *pid_name, pid_t *group, int64_t *rss token = strtok_r(NULL, " ", &save); field++; } - return have_group && have_rss; + return have_group && have_rss ? CBM_LINUX_STAT_OK : CBM_LINUX_STAT_UNREADABLE; } static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, @@ -687,17 +784,18 @@ static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subproces uint64_t total = 0; int measured = 0; - bool root_failed = false; + bool failed = false; struct dirent *entry; - while ((entry = readdir(proc)) != NULL) { + while (!failed && (entry = readdir(proc)) != NULL) { if (!cbm_proc_pid_name(entry->d_name)) { continue; } pid_t group = 0; int64_t pages = 0; bool root_entry = strtol(entry->d_name, NULL, 10) == (long)process->pid; - if (!cbm_linux_proc_stat(entry->d_name, &group, &pages)) { - root_failed = root_failed || root_entry; + cbm_linux_stat_t status = cbm_linux_proc_stat(entry->d_name, root_entry, &group, &pages); + if (status != CBM_LINUX_STAT_OK) { + failed = root_entry || status != CBM_LINUX_STAT_GONE; continue; } if (group != process->pgid || pages < 0) { @@ -711,7 +809,7 @@ static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subproces measured++; } (void)closedir(proc); - if (root_failed) { + if (failed) { return CBM_PROC_TREE_RSS_ERROR; } if (measured == 0) { diff --git a/src/foundation/subprocess.h b/src/foundation/subprocess.h index 94a8a2ed01..a5b44474a3 100644 --- a/src/foundation/subprocess.h +++ b/src/foundation/subprocess.h @@ -159,8 +159,9 @@ cbm_proc_poll_t cbm_subprocess_poll(cbm_subprocess_t *process, cbm_proc_result_t * On macOS this is phys_footprint (what the OS holds against the process); * elsewhere it is RSS. OK returns an overflow-safe byte total, EMPTY means * the owned tree currently has no observable members, and ERROR means no - * trustworthy measurement could be obtained. Individual processes that exit - * during enumeration are ignored. */ + * trustworthy measurement could be obtained. A member that exits between the + * listing and its measurement is ignored (the platform reports it gone); any + * other member that cannot be measured is ERROR, never a smaller total. */ cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_bytes(cbm_subprocess_t *process, uint64_t *rss_bytes); bool cbm_subprocess_root_running(const cbm_subprocess_t *process); @@ -243,6 +244,10 @@ void cbm_subprocess_hold_parent_until_child_exits_for_testing(int spawns); * waitid does not report it yet. Test builds only. */ void cbm_subprocess_observe_exiting_child_for_testing(int spawns); uint64_t cbm_subprocess_rss_sum_for_testing(const uint64_t *values, size_t count); +/* Make the tree probe's measurement of every NON-root member fail with + * platform_error (errno on POSIX, GetLastError() on Windows), as a live or an + * exiting member would; 0 restores the real calls. Test builds only. */ +void cbm_subprocess_tree_rss_fault_members_for_testing(int platform_error); #endif /* How the POSIX fork+exec child closed its inherited descriptors (#1484). */ diff --git a/tests/test_subprocess.c b/tests/test_subprocess.c index 1c5039e87a..e214a2c36f 100644 --- a/tests/test_subprocess.c +++ b/tests/test_subprocess.c @@ -813,6 +813,17 @@ TEST(subprocess_spawn_returns_while_child_is_running) { * Waits below have no deadline: they end when the descendant is ready or the * tree stops running, so the outcome, not a timer, decides the verdict. */ +#ifdef _WIN32 +#define RSS_TREE_FAULT_GONE ERROR_INVALID_PARAMETER /* no process with that id any more */ +#define RSS_TREE_FAULT_LIVE ERROR_ACCESS_DENIED /* a member we may not open */ +#elif defined(__APPLE__) +#define RSS_TREE_FAULT_GONE ESRCH +#define RSS_TREE_FAULT_LIVE EPERM +#else +#define RSS_TREE_FAULT_GONE ENOENT +#define RSS_TREE_FAULT_LIVE EACCES +#endif + #define RSS_TREE_MIB UINT64_C(1048576) typedef struct { @@ -866,8 +877,9 @@ static bool rss_tree_finish(cbm_subprocess_t *process, rss_tree_sample_t *sample return result.tree_quiesced; } -/* Measure one tree whose descendant holds `mib` MiB. */ -static rss_tree_sample_t rss_tree_sample(int mib) { +/* Measure one tree whose descendant holds `mib` MiB, with every non-root + * member's measurement failing with `member_fault` (0 = real calls). */ +static rss_tree_sample_t rss_tree_sample(int mib, int member_fault) { rss_tree_sample_t sample = {.status = CBM_PROC_TREE_RSS_ERROR, .final_status = CBM_PROC_TREE_RSS_ERROR, .final_rss = UINT64_MAX}; @@ -891,7 +903,9 @@ static rss_tree_sample_t rss_tree_sample(int mib) { if (sample.spawned) { sample.ready = rss_tree_wait_ready(ready_path, process); if (sample.ready) { + cbm_subprocess_tree_rss_fault_members_for_testing(member_fault); sample.status = cbm_subprocess_tree_rss_bytes(process, &sample.rss); + cbm_subprocess_tree_rss_fault_members_for_testing(0); } sample.quiesced = rss_tree_finish(process, &sample); } @@ -904,8 +918,8 @@ static rss_tree_sample_t rss_tree_sample(int mib) { * the root. 128 MiB touched against a 96 MiB floor leaves room for allocator * and page-rounding noise and none for a root-only total. */ TEST(subprocess_tree_rss_measures_contained_descendants) { - rss_tree_sample_t idle = rss_tree_sample(0); - rss_tree_sample_t loaded = rss_tree_sample(128); + rss_tree_sample_t idle = rss_tree_sample(0, 0); + rss_tree_sample_t loaded = rss_tree_sample(128, 0); if (loaded.status == CBM_PROC_TREE_RSS_OK && idle.status == CBM_PROC_TREE_RSS_OK) { printf(" tree rss: idle=%llu MiB loaded=%llu MiB\n", (unsigned long long)(idle.rss / RSS_TREE_MIB), @@ -926,6 +940,29 @@ TEST(subprocess_tree_rss_measures_contained_descendants) { PASS(); } +/* A live member the platform will not let us measure must fail the probe: a + * total without it would under-report the tree. */ +TEST(subprocess_tree_rss_fails_when_a_live_member_cannot_be_measured) { + rss_tree_sample_t sample = rss_tree_sample(0, RSS_TREE_FAULT_LIVE); + ASSERT_TRUE(sample.spawned); + ASSERT_TRUE(sample.ready); + ASSERT_EQ(sample.status, CBM_PROC_TREE_RSS_ERROR); + ASSERT_TRUE(sample.quiesced); + PASS(); +} + +/* A member that exited between the listing and its measurement holds nothing: + * it is skipped and the rest of the tree (here the root) is still measured. */ +TEST(subprocess_tree_rss_skips_a_member_that_exited_mid_probe) { + rss_tree_sample_t sample = rss_tree_sample(0, RSS_TREE_FAULT_GONE); + ASSERT_TRUE(sample.spawned); + ASSERT_TRUE(sample.ready); + ASSERT_EQ(sample.status, CBM_PROC_TREE_RSS_OK); + ASSERT_TRUE(sample.rss > 0); + ASSERT_TRUE(sample.quiesced); + PASS(); +} + TEST(subprocess_natural_completion_is_cached_across_polls) { #ifdef _WIN32 SKIP_PLATFORM("POSIX /bin/sh completion-cache probe; native Job Object coverage pending"); @@ -2775,6 +2812,8 @@ SUITE(subprocess) { RUN_TEST(subprocess_run_null_bin_rejected); RUN_TEST(subprocess_spawn_returns_while_child_is_running); RUN_TEST(subprocess_tree_rss_measures_contained_descendants); + RUN_TEST(subprocess_tree_rss_fails_when_a_live_member_cannot_be_measured); + RUN_TEST(subprocess_tree_rss_skips_a_member_that_exited_mid_probe); RUN_TEST(subprocess_natural_completion_is_cached_across_polls); RUN_TEST(subprocess_cancel_is_idempotent_and_kills_ignoring_tree); RUN_TEST(subprocess_quiet_timeout_kills_ignoring_tree); From 91070cb2f0309002cca23c69fc3edf4e87feb2f6 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 2 Nov 2026 01:11:32 +0100 Subject: [PATCH 19/27] test(lint-mem): run the warning-suffix gate test on every leg scripts/test.sh lists every shell contract test explicitly and does not glob tests/, so tests/test_lint_mem_gate.sh never ran on any leg. Register it as Step 0e4, next to the other gate contract (Step 0e3), in the same two-line shape as its neighbours. Step 0 runs in every default scripts/test.sh leg (macOS, Linux, Windows) and in the PR contracts-only job. The wiring, Windows included, is a maintainer decision (R1). Make the test portable to the Windows leg. There python3 is a native MSYS2 build: MSYS converts path arguments for it, but never the diagnostic text on stdin, so the absolute fixture path inside the finding does not resolve there, and a C:/ path would not parse (the file field of FINDING_RE ends at the first colon). The gate now runs from inside the fixture directory, in a subshell, with relative paths. CRLF from a native python3 is stripped from the --hash output, the whitelist is written once the hash is known (no placeholder rewrite), the gate exit status is asserted, and every failed assertion is reported together with the gate output. Refs #2442 Signed-off-by: Martin Vogel --- scripts/test.sh | 3 ++ tests/test_lint_mem_gate.sh | 71 +++++++++++++++++++++++++++++-------- 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index cdc6d3e1d4..1b4873c9f0 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -267,6 +267,9 @@ bash "$ROOT/tests/test_activation_diagnostic_contract.sh" echo "=== Step 0e3: security gate fail-closed contract ===" bash "$ROOT/tests/test_security_gate_fail_closed.sh" +echo "=== Step 0e4: memory-analyzer gate warning-suffix contract (#2442) ===" +bash "$ROOT/tests/test_lint_mem_gate.sh" + echo "=== Step 0f: tree-sitter runtime Makefile dependencies ===" bash "$ROOT/tests/test_makefile_ts_runtime_dependencies.sh" diff --git a/tests/test_lint_mem_gate.sh b/tests/test_lint_mem_gate.sh index 05c2bd4a3a..c6a655222c 100755 --- a/tests/test_lint_mem_gate.sh +++ b/tests/test_lint_mem_gate.sh @@ -1,20 +1,56 @@ #!/usr/bin/env bash -# Regression contract for clang-tidy's `,-warnings-as-errors` diagnostic suffix. +# Regression contract for clang-tidy's `,-warnings-as-errors` diagnostic suffix +# (#2442). With WarningsAsErrors enabled, clang-tidy prints an analyzer check as +# [clang-analyzer-,-warnings-as-errors]; a gate that only accepts +# [] drops every such line and reports a clean gate. A suffixed finding +# WITH an argued whitelist entry must be counted under its plain check name and +# suppressed (exit 0, "1 argued false positive"). Every assertion is reported, +# so a regression names what broke. +# +# The gate runs from inside the fixture directory with RELATIVE paths. On the +# Windows leg python3 is a native MSYS2 build: MSYS converts path ARGUMENTS for +# it, but never the diagnostic text on stdin, so an absolute /c/... path in a +# finding would not resolve there, and a C:/... one would not parse (the file +# field ends at the first colon). set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GATE="$ROOT/scripts/lint-mem-gate.py" FIX="$(mktemp -d "${TMPDIR:-/tmp}/lint-mem-gate.XXXXXX")" trap 'rm -rf "$FIX"' EXIT +failures=0 +fail() { + echo "FAIL: $*" >&2 + failures=$((failures + 1)) +} + +show() { + sed 's/^/ | /' "$1" >&2 +} + +# A subshell, so the test's own shell never sits inside $FIX (Windows cannot +# remove a directory that is a process's working directory). +run_gate() { + (cd "$FIX" && LINT_MEM_WHITELIST=whitelist.txt python3 "$GATE" "$@") +} + cat > "$FIX/sample.c" <<'EOF' int sample(int *value) { return *value; } EOF +# A native Windows python3 writes CRLF; strip it before the hash is reused. +HASH="$(run_gate --hash sample.c sample | tr -d '\r' | awk '/^segment-sha256:/ {print $2}')" +if [[ ! "$HASH" =~ ^[0-9a-f]{64}$ ]]; then + echo "FAIL: lint-mem-gate.py --hash sample.c sample printed no segment hash (got '$HASH')" >&2 + exit 1 +fi + cat > "$FIX/whitelist.txt" < "$FIX/stderr" +# Whitelisted: counted under the plain check name, then suppressed. +rc=0 +printf '%s\n' "sample.c:2:12: warning: possible null dereference $SUFFIX" \ + | run_gate > /dev/null 2> "$FIX/whitelisted.err" || rc=$? +if [ "$rc" -ne 0 ]; then + fail "a whitelisted suffixed finding left the gate at exit $rc, expected 0" + show "$FIX/whitelisted.err" +fi +if ! grep -q 'memory gate clean (1 argued false positive' "$FIX/whitelisted.err"; then + fail "a whitelisted suffixed finding was not counted as 1 argued false positive" + show "$FIX/whitelisted.err" +fi -grep -q 'memory gate clean (1 argued false positive' "$FIX/stderr" +if [ "$failures" -ne 0 ]; then + echo "lint-mem-gate warning suffix contract: $failures assertion(s) failed" >&2 + exit 1 +fi echo 'lint-mem-gate warning suffix contract passed' From 24871eed7b62ea94e1629570160b083d4f445255 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 2 Nov 2026 01:11:51 +0100 Subject: [PATCH 20/27] test(lint-mem): an unwhitelisted suffixed finding must fail the gate The test only covered a whitelisted finding. The defect in #2442 is the other direction: on main clang-tidy reported findings that carry the ,-warnings-as-errors suffix, the old FINDING_RE matched none of them, and the gate reported clean. Pin that direction too: a second fixture function with no whitelist entry, a suffixed finding inside it, and the gate must exit 1 and report the finding under its plain check name ([clang-analyzer-core.NullDereference], the whitelist key form) as an entry missing for unlisted(). With the suffix group of FINDING_RE reverted, this case exits 0 with "memory gate clean" and the test fails, naming all three assertions. A parser that accepts the line but keeps the suffix inside the check name fails it as well. The negative assertion is a maintainer decision (R1). Refs #2442 Signed-off-by: Martin Vogel --- tests/test_lint_mem_gate.sh | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/tests/test_lint_mem_gate.sh b/tests/test_lint_mem_gate.sh index c6a655222c..9178bca564 100755 --- a/tests/test_lint_mem_gate.sh +++ b/tests/test_lint_mem_gate.sh @@ -2,10 +2,15 @@ # Regression contract for clang-tidy's `,-warnings-as-errors` diagnostic suffix # (#2442). With WarningsAsErrors enabled, clang-tidy prints an analyzer check as # [clang-analyzer-,-warnings-as-errors]; a gate that only accepts -# [] drops every such line and reports a clean gate. A suffixed finding -# WITH an argued whitelist entry must be counted under its plain check name and -# suppressed (exit 0, "1 argued false positive"). Every assertion is reported, -# so a regression names what broke. +# [] drops every such line and reports a clean gate. Both directions are +# pinned, and every assertion is reported, so a regression names what broke: +# 1. a suffixed finding WITH an argued whitelist entry is counted under its +# plain check name and suppressed (exit 0, "1 argued false positive"); +# 2. a suffixed finding WITHOUT an entry fails the gate (exit 1) and is +# reported under its plain check name. This is the negative assertion: a +# parser that drops the suffixed line reports clean (exit 0) here. +# Reverting the suffix group in FINDING_RE (scripts/lint-mem-gate.py) turns +# both red. # # The gate runs from inside the fixture directory with RELATIVE paths. On the # Windows leg python3 is a native MSYS2 build: MSYS converts path ARGUMENTS for @@ -39,6 +44,10 @@ cat > "$FIX/sample.c" <<'EOF' int sample(int *value) { return *value; } + +int unlisted(int *value) { + return *value; +} EOF # A native Windows python3 writes CRLF; strip it before the hash is reused. @@ -60,7 +69,7 @@ EOF SUFFIX='[clang-analyzer-core.NullDereference,-warnings-as-errors]' -# Whitelisted: counted under the plain check name, then suppressed. +# 1. Whitelisted: counted under the plain check name, then suppressed. rc=0 printf '%s\n' "sample.c:2:12: warning: possible null dereference $SUFFIX" \ | run_gate > /dev/null 2> "$FIX/whitelisted.err" || rc=$? @@ -73,8 +82,24 @@ if ! grep -q 'memory gate clean (1 argued false positive' "$FIX/whitelisted.err" show "$FIX/whitelisted.err" fi +# 2. Not whitelisted: the gate must fail and name the finding. +rc=0 +printf '%s\n' "sample.c:6:12: warning: possible null dereference $SUFFIX" \ + | run_gate > /dev/null 2> "$FIX/unlisted.err" || rc=$? +if [ "$rc" -ne 1 ]; then + fail "an unwhitelisted suffixed finding left the gate at exit $rc, expected 1" \ + "(exit 0 = the suffixed line was parsed as no finding, #2442)" + show "$FIX/unlisted.err" +fi +if ! grep -qF 'sample.c:6: possible null dereference [clang-analyzer-core.NullDereference]' "$FIX/unlisted.err" || + ! grep -qF 'no whitelist entry for unlisted()' "$FIX/unlisted.err" || + ! grep -qF '1 finding(s) unaccounted for' "$FIX/unlisted.err"; then + fail "the gate did not report the unwhitelisted suffixed finding under its plain check name" + show "$FIX/unlisted.err" +fi + if [ "$failures" -ne 0 ]; then echo "lint-mem-gate warning suffix contract: $failures assertion(s) failed" >&2 exit 1 fi -echo 'lint-mem-gate warning suffix contract passed' +echo 'lint-mem-gate warning suffix contract passed (whitelisted + unwhitelisted suffixed findings)' From 435a88be6271a392f22487e41b5e2f5867e78313 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 2 Nov 2026 18:12:22 +0100 Subject: [PATCH 21/27] fix(index): coordinate publication between the worker and its supervisor A worker resource limit is enforced by the supervisor, but only the worker replaces the serving database, and it does so before the persistence artifact export. A limit observed in that window stopped a worker that had already published, and the response still said serving_index_preserved=true while the previous index had been replaced. Publication and resource termination now coordinate through a per-attempt publication fence: a never-created path next to a reserved unique file in the private worker directory, which both sides claim by creating it exclusively (cbm_fopen "wbx": O_EXCL on POSIX, CREATE_NEW on Windows), so exactly one of them can win. - The worker claims it through a new pipeline publication gate that cbm_pipeline_run calls once, after the last pre-publication check and before the serving database is touched. Losing means the supervisor already stopped the attempt for a limit: the run aborts before publication and the previous generation stays in place. - The supervisor claims it before it terminates the worker for a limit or a failed RSS probe. Losing means the worker already committed to publishing: the limit is no longer enforced for that attempt, which runs to completion, and index.supervisor.resource_limit_after_publication is logged instead. A claim that can be neither won nor lost is retried on the next poll rather than risk stopping a committed worker. - At terminal the supervisor settles the fence into result.publication_committed. An unquiesced tree keeps the fence, so a surviving member can never publish after the report. serving_index_preserved now describes the request, not only its last attempt: a servable index existed when the request began, no attempt of the request committed to publishing (an earlier attempt can publish and then crash into recovery), and that index is still servable. The direct path and the daemon job both fold every attempt into these request facts; cbm_mcp_index_worker_resource_response takes them as a third argument, and NULL claims nothing. The fence path travels as an optional last pair of the exact worker grammar (--index-worker-publish-fence), accepted by the worker argv parser and by the daemon bootstrap role check, stripped by the CLI and installed by both worker entries. Limits stay opt-in: without index_max_rss_mb or index_max_duration_seconds there is no fence, no extra argv, no claim and no extra database read. Graph content is unchanged either way. RED (c78c077a plus only the new end-to-end test): a FIFO at the artifact temp path holds the real worker after the rename, and the duration clock crosses its limit only once the serving database has been replaced. The supervisor stops the worker there and answers resource_limit_exceeded with serving_index_preserved=true over a replaced database. GREEN: the worker claimed first, the supervisor logs the limit as not enforced, and the released worker finishes and reports the published index. Signed-off-by: Martin Vogel --- docs/CONFIGURATION.md | 2 +- docs/INDEX_RESOURCE_LIMITS.md | 17 +- src/daemon/application.c | 24 ++- src/daemon/bootstrap.c | 6 + src/main.c | 4 + src/mcp/index_supervisor.c | 214 ++++++++++++++++++++++++- src/mcp/index_supervisor.h | 37 +++++ src/mcp/mcp.c | 70 +++++++-- src/mcp/mcp_internal.h | 21 ++- src/pipeline/pipeline.c | 22 +++ src/pipeline/pipeline.h | 11 ++ tests/test_index_supervisor.c | 225 ++++++++++++++++++++++++++- tests/test_main.c | 7 +- tests/test_mcp.c | 285 ++++++++++++++++++++++++++++++++++ tests/test_pipeline.c | 67 ++++++++ 15 files changed, 992 insertions(+), 20 deletions(-) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index a02459b4ab..2f0d6717a5 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -94,7 +94,7 @@ Current keys: | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | | `index_max_rss_mb` | `off` | Optional maximum charged memory in MiB for the complete contained index-worker process tree (`64..1048576`). On macOS this is phys_footprint, not resident size. | -| `index_max_duration_seconds` | `off` | Optional maximum wall-clock duration in seconds for the whole index request, including crash/hang recovery (`1..86400`). | +| `index_max_duration_seconds` | `off` | Optional maximum wall-clock duration in seconds for the whole index request, including crash/hang recovery (`1..86400`). Like `index_max_rss_mb`, it stops a worker only before it starts to publish the new index; see [INDEX_RESOURCE_LIMITS.md](INDEX_RESOURCE_LIMITS.md). | For a watched project, `index_status` names the polling method in `watch.strategy`: `git` (HEAD and dirty-state polling), `tree` (a non-git root polled by the diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index 11fcf4a3d4..d76b162d76 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -58,6 +58,18 @@ terminal only after the tree is quiescent or a bounded containment failure is explicitly surfaced. Resource termination is not retried and does not quarantine a source file. +A worker limit never stops a worker after it has started to publish. The +supervisor enforces the limits, but only the worker replaces the serving +database, so the two coordinate through a per-attempt publication claim that +exactly one of them can win. The worker claims it immediately before it +replaces the serving database; the supervisor claims it before it stops the +worker for a limit. If the limit is reached first, the worker can no longer +publish and the previous index stays untouched. If the worker claimed first, +the limits no longer apply to that attempt: the new index is published and the +rest of the attempt, including the persistence artifact export, runs to +completion. The supervisor then logs `index.supervisor.resource_limit_after_publication` +instead of stopping it. With both worker limits `off` there is no claim. + If RSS is enabled and three consecutive probes cannot obtain any trustworthy tree measurement while the root worker is still running, CBM fails closed with `code=resource_probe_failed`. A probe is untrustworthy as soon as one member of @@ -102,7 +114,10 @@ Worker limit failures use the same shape with `stage=worker`, `resource=rss_bytes` and `unit=bytes`, or `resource=duration_ms` and `unit=milliseconds`. RSS measurement failures use `code=resource_probe_failed` and omit `observed`, `limit`, and `unit` because no trustworthy observation was -available. +available. For a worker failure `serving_index_preserved` describes the whole +request: it is true only when a servable index existed when the request began, +no attempt of the request (including an earlier one that crashed and was +recovered) claimed publication, and that index still serves. ## Trust and compatibility diff --git a/src/daemon/application.c b/src/daemon/application.c index 0f696d59c4..207735e04d 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -160,6 +160,10 @@ struct cbm_daemon_application_job { bool async_owned; bool worker_started; uint64_t request_started_ms; + /* What a worker resource failure of this job is reported against; set + * before the first attempt, folded over every attempt (mcp_internal.h). */ + bool request_facts_begun; + cbm_mcp_index_request_facts_t request_facts; cbm_daemon_application_job_t *next; }; @@ -1346,6 +1350,17 @@ static void application_cancelled_result(cbm_index_worker_result_t *result) { result->tree_quiesced = true; } +/* Only an enabled worker limit can end a job in a worker resource failure, the + * one response the request facts feed; without one nothing is read. */ +static void application_job_request_facts_begin(cbm_daemon_application_job_t *job) { + cbm_index_resource_policy_t policy; + char error[CBM_SZ_256] = {0}; + if (cbm_mcp_index_policy_from_internal_args(job->args_json, &policy, error, sizeof(error)) && + cbm_index_policy_worker_enabled(&policy)) { + cbm_mcp_index_request_facts_begin(job->args_json, &job->request_facts); + } +} + static application_attempt_status_t application_job_run_attempt(cbm_daemon_application_job_t *job, const char *marker_path, const char *quarantine_path, @@ -1382,6 +1397,10 @@ static application_attempt_status_t application_job_run_attempt(cbm_daemon_appli if (job->request_started_ms == 0) { job->request_started_ms = cbm_index_worker_now_ms(); } + if (!job->request_facts_begun) { + job->request_facts_begun = true; + application_job_request_facts_begin(job); + } application_tmp_lock(); int start_result = application->worker_ops.start( application->worker_ops.context, job->args_json, memory_budget_bytes, marker_path, @@ -1573,6 +1592,7 @@ static application_attempt_decision_t application_consume_attempt( application_attempt_disposition(job, attempt); application_record_attempt(attempt, &execution->last_result, &execution->have_last_result, execution->last_log); + cbm_mcp_index_request_facts_record(&job->request_facts, &attempt->result); if (disposition == CBM_MCP_SUPERVISED_RESULT_SUCCESS) { execution->response = attempt->result.response; attempt->result.response = NULL; @@ -1581,8 +1601,8 @@ static application_attempt_decision_t application_consume_attempt( return APPLICATION_ATTEMPT_DECISION_SUCCESS; } if (disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { - execution->response = - cbm_mcp_index_worker_resource_response(job->args_json, &attempt->result); + execution->response = cbm_mcp_index_worker_resource_response( + job->args_json, &attempt->result, &job->request_facts); application_attempt_free(attempt); return APPLICATION_ATTEMPT_DECISION_STOP; } diff --git a/src/daemon/bootstrap.c b/src/daemon/bootstrap.c index cb384cd20c..617b9c06fc 100644 --- a/src/daemon/bootstrap.c +++ b/src/daemon/bootstrap.c @@ -141,6 +141,12 @@ static bool bootstrap_worker_argv_exact(int argc, char *const argv[]) { } next += 2; } + if (next < argc && bootstrap_arg_is(argv[next], "--index-worker-publish-fence")) { + if (next + 1 >= argc || !argv[next + 1] || !argv[next + 1][0]) { + return false; + } + next += 2; + } return next == argc; } diff --git a/src/main.c b/src/main.c index 54bad875eb..82be46e030 100644 --- a/src/main.c +++ b/src/main.c @@ -897,6 +897,8 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo const char *worker_marker = cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_MARKER_ARG); const char *worker_quarantine = cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_QUARANTINE_ARG); + const char *worker_publish_fence = + cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_PUBLISH_FENCE_ARG); if (index_worker) { /* The graph lives on this process's heaps: SQLite gets heaps of its * own here, and only here (cbm_sqlite_dedicated_heap). */ @@ -906,6 +908,7 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo cbm_index_set_worker_role_options(index_worker, response_out, worker_single_thread, worker_marker, worker_quarantine, cbm_index_worker_memory_budget_bytes()); + cbm_index_set_worker_publish_fence(index_worker ? worker_publish_fence : NULL); if (argc < MAIN_MIN_ARGC) { (void)fprintf(stderr, CLI_USAGE); @@ -3188,6 +3191,7 @@ int main(int argc, char **argv) { cbm_index_set_worker_role_options(true, invocation.response_out, invocation.single_thread, invocation.marker_file, invocation.quarantine_file, invocation.memory_budget_bytes); + cbm_index_set_worker_publish_fence(invocation.publish_fence_file); #ifndef _WIN32 /* Split into three ordered steps rather than one condition, because the * ORDER is load-bearing and the middle step only exists in test builds: diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index e84dff7557..c8c4224ad0 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -92,6 +92,63 @@ size_t cbm_index_worker_memory_budget_bytes(void) { return g_worker_memory_budget_bytes; } +/* ── Publication fence (see index_supervisor.h) ───────────────────── */ + +typedef enum { + INDEX_PUBLICATION_CLAIMED = 0, /* this caller created the fence file */ + INDEX_PUBLICATION_TAKEN, /* the other side created it first */ + INDEX_PUBLICATION_CLAIM_ERROR, /* neither could be established */ +} index_publication_claim_t; + +/* Exclusive creation is the single atomic decision both processes race for: + * "x" is O_CREAT|O_EXCL on POSIX and CREATE_NEW on Windows, so exactly one + * creator succeeds and the other gets EEXIST. A deletion-based claim would + * be ambiguous on Windows, where a delete-pending file reports access denied. */ +static index_publication_claim_t index_publication_claim(const char *fence_file) { + if (!fence_file || !fence_file[0]) { + return INDEX_PUBLICATION_CLAIM_ERROR; + } + errno = 0; + FILE *claim = cbm_fopen(fence_file, "wbx"); + if (claim) { + (void)fclose(claim); + return INDEX_PUBLICATION_CLAIMED; + } + return errno == EEXIST ? INDEX_PUBLICATION_TAKEN : INDEX_PUBLICATION_CLAIM_ERROR; +} + +static char g_worker_publish_fence[CBM_SZ_4K] = {0}; +static bool g_worker_publication_claimed = false; + +void cbm_index_set_worker_publish_fence(const char *fence_file) { + if (fence_file && fence_file[0]) { + (void)snprintf(g_worker_publish_fence, sizeof(g_worker_publish_fence), "%s", fence_file); + } else { + g_worker_publish_fence[0] = '\0'; + } + g_worker_publication_claimed = false; +} + +const char *cbm_index_worker_publish_fence(void) { + return g_worker_publish_fence[0] ? g_worker_publish_fence : NULL; +} + +bool cbm_index_worker_claim_publication(void) { + if (!g_worker_publish_fence[0] || g_worker_publication_claimed) { + return true; + } + index_publication_claim_t claim = index_publication_claim(g_worker_publish_fence); + if (claim == INDEX_PUBLICATION_CLAIMED) { + g_worker_publication_claimed = true; + cbm_log_info("index.worker.publication_claimed", "fence", g_worker_publish_fence); + return true; + } + cbm_log_warn("index.worker.publication_refused", "reason", + claim == INDEX_PUBLICATION_TAKEN ? "resource_limit" : "claim_failed", "fence", + g_worker_publish_fence); + return false; +} + /* Worker log startup header — see index_supervisor.h. */ static atomic_flag g_worker_log_begun = ATOMIC_FLAG_INIT; @@ -250,6 +307,13 @@ cbm_index_worker_argv_status_t cbm_index_worker_parse_process_argv( parsed.quarantine_file = argv[next + 1]; next += 2; } + if (next < argc && argv[next] && strcmp(argv[next], CBM_INDEX_WORKER_PUBLISH_FENCE_ARG) == 0) { + if (next + 1 >= argc || !argv[next + 1] || !argv[next + 1][0]) { + return CBM_INDEX_WORKER_ARGV_INVALID; + } + parsed.publish_fence_file = argv[next + 1]; + next += 2; + } if (next != argc) { return CBM_INDEX_WORKER_ARGV_INVALID; } @@ -477,7 +541,7 @@ static void worker_log_last_phase(const char *log_path, char *out, size_t out_si enum { INDEX_WORKER_PATH_CAP = CBM_SZ_4K, - INDEX_WORKER_ARGV_CAP = 17, + INDEX_WORKER_ARGV_CAP = 19, INDEX_WORKER_SYNC_POLL_NS = 10000000, INDEX_WORKER_RELAY_LINES_PER_POLL = 64, INDEX_WORKER_RELAY_BYTES_PER_POLL = 64 * 1024, @@ -492,6 +556,14 @@ typedef enum { INDEX_WORKER_TERMINATION_RESOURCE, } index_worker_termination_reason_t; +/* Who owns the publication fence of one attempt. Only the polling owner thread + * reads or writes it. */ +typedef enum { + INDEX_WORKER_PUBLICATION_OPEN = 0, /* unclaimed, or no fence at all */ + INDEX_WORKER_PUBLICATION_SUPERVISOR, + INDEX_WORKER_PUBLICATION_WORKER, +} index_worker_publication_owner_t; + #ifdef CBM_ENABLE_TEST_SEAMS static cbm_index_supervisor_clock_fn g_resource_clock; static cbm_index_supervisor_rss_fn g_resource_rss; @@ -545,11 +617,24 @@ struct cbm_index_worker_handle { uint64_t last_rss_probe_ms; unsigned int rss_probe_failures; bool rss_probe_started; + /* Publication fence: a never-created path next to a reserved unique file + * (both removed at terminal). Empty when no worker limit is enabled. */ + char publish_fence_reserve[INDEX_WORKER_PATH_CAP]; + char publish_fence_path[INDEX_WORKER_PATH_CAP]; + index_worker_publication_owner_t publication_owner; + bool publication_claim_error_logged; atomic_int termination_reason; atomic_bool terminal; cbm_index_worker_result_t result; }; +#ifdef CBM_ENABLE_TEST_SEAMS +const char *cbm_index_worker_publish_fence_path_for_testing( + const cbm_index_worker_handle_t *handle) { + return handle && handle->publish_fence_path[0] ? handle->publish_fence_path : NULL; +} +#endif + /* The generic subprocess supervisor tails logs in bounded batches to preserve * its nonblocking poll contract. A short-lived worker can therefore become * terminal with more completed lines still on disk. Keep the request callback @@ -801,9 +886,58 @@ size_t cbm_index_worker_job_memory_limit(size_t memory_budget_bytes) { return memory_budget_bytes > SIZE_MAX - headroom ? SIZE_MAX : memory_budget_bytes + headroom; } +/* Claim the publication fence before stopping the worker for a resource limit + * (see index_supervisor.h). True when the termination may proceed. */ +static bool worker_claim_publication_for_limit(cbm_index_worker_handle_t *handle, + cbm_index_resource_t resource, uint64_t observed, + uint64_t limit, bool probe_failed) { + if (!handle->publish_fence_path[0] || + handle->publication_owner == INDEX_WORKER_PUBLICATION_SUPERVISOR) { + return true; + } + if (handle->publication_owner == INDEX_WORKER_PUBLICATION_WORKER) { + return false; + } + index_publication_claim_t claim = index_publication_claim(handle->publish_fence_path); + if (claim == INDEX_PUBLICATION_CLAIMED) { + handle->publication_owner = INDEX_WORKER_PUBLICATION_SUPERVISOR; + return true; + } + if (claim == INDEX_PUBLICATION_TAKEN) { + /* The worker committed to publishing before the limit was observed. + * Stopping it now could leave a new generation serving behind a report + * that the previous index was preserved, so this attempt is no longer + * limited; the crossing is recorded here instead. */ + handle->publication_owner = INDEX_WORKER_PUBLICATION_WORKER; + char observed_text[CBM_SZ_32]; + char limit_text[CBM_SZ_32]; + (void)snprintf(observed_text, sizeof(observed_text), "%llu", (unsigned long long)observed); + (void)snprintf(limit_text, sizeof(limit_text), "%llu", (unsigned long long)limit); + cbm_log_warn("index.supervisor.resource_limit_after_publication", "resource", + cbm_index_resource_name(resource), "observed", + probe_failed ? "unmeasured" : observed_text, "limit", limit_text, "action", + "not_enforced", "log", handle->log_path); + return false; + } + /* Neither side holds the fence and it could not be created. Terminating + * now could race a later successful claim by the worker, so the decision + * is retried on the next poll instead. */ + if (!handle->publication_claim_error_logged) { + handle->publication_claim_error_logged = true; + char errno_text[CBM_SZ_32]; + (void)snprintf(errno_text, sizeof(errno_text), "%d", errno); + cbm_log_error("index.supervisor.publication_claim_failed", "errno", errno_text, "action", + "retry_next_poll", "log", handle->log_path); + } + return false; +} + static void worker_request_resource_termination(cbm_index_worker_handle_t *handle, cbm_index_resource_t resource, uint64_t observed, uint64_t limit, bool probe_failed) { + if (!worker_claim_publication_for_limit(handle, resource, observed, limit, probe_failed)) { + return; + } int expected = INDEX_WORKER_TERMINATION_NONE; if (!atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, INDEX_WORKER_TERMINATION_RESOURCE, @@ -824,7 +958,9 @@ static void worker_request_resource_termination(cbm_index_worker_handle_t *handl } static bool worker_check_resource_limits(cbm_index_worker_handle_t *handle) { - if (atomic_load_explicit(&handle->termination_reason, memory_order_acquire) != + /* A worker that committed to publishing is no longer resource-limited. */ + if (handle->publication_owner == INDEX_WORKER_PUBLICATION_WORKER || + atomic_load_explicit(&handle->termination_reason, memory_order_acquire) != INDEX_WORKER_TERMINATION_NONE || cbm_subprocess_termination_pending(handle->process) || !cbm_subprocess_supervision_active(handle->process)) { @@ -877,6 +1013,56 @@ static bool worker_check_resource_limits(cbm_index_worker_handle_t *handle) { return rss_probe_failed; } +/* Reserve a unique private name and derive the fence path from it. The fence + * itself is never created here: creating it IS the claim. */ +static bool worker_publish_fence_create(cbm_index_worker_handle_t *handle) { + if (!worker_unique_file(handle->publish_fence_reserve, sizeof(handle->publish_fence_reserve), + "publish")) { + return false; + } + int written = snprintf(handle->publish_fence_path, sizeof(handle->publish_fence_path), + "%s.claim", handle->publish_fence_reserve); + if (written <= 0 || written >= (int)sizeof(handle->publish_fence_path)) { + (void)cbm_unlink(handle->publish_fence_reserve); + handle->publish_fence_reserve[0] = '\0'; + handle->publish_fence_path[0] = '\0'; + return false; + } + return true; +} + +static void worker_publish_fence_remove(cbm_index_worker_handle_t *handle) { + if (handle->publish_fence_path[0]) { + (void)cbm_unlink(handle->publish_fence_path); + } + if (handle->publish_fence_reserve[0]) { + (void)cbm_unlink(handle->publish_fence_reserve); + } +} + +/* Settle who held the fence once the attempt is terminal. Claiming an unowned + * fence now also bars any later claim, so a stray member of an unquiesced tree + * can never publish after this report. */ +static bool worker_publication_committed_at_terminal(cbm_index_worker_handle_t *handle) { + if (!handle->publish_fence_path[0] || + handle->publication_owner == INDEX_WORKER_PUBLICATION_SUPERVISOR) { + return false; + } + if (handle->publication_owner == INDEX_WORKER_PUBLICATION_WORKER) { + return true; + } + index_publication_claim_t claim = index_publication_claim(handle->publish_fence_path); + if (claim == INDEX_PUBLICATION_CLAIMED) { + handle->publication_owner = INDEX_WORKER_PUBLICATION_SUPERVISOR; + return false; + } + if (claim == INDEX_PUBLICATION_TAKEN) { + handle->publication_owner = INDEX_WORKER_PUBLICATION_WORKER; + } + /* Taken: the worker committed. Unreadable: preservation cannot be proven. */ + return true; +} + static int worker_start_internal(const char *args_json, size_t memory_budget_bytes, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, @@ -935,6 +1121,19 @@ static int worker_start_internal(const char *args_json, size_t memory_budget_byt free(handle); return -1; } + /* Opt-in: only an enabled worker limit can stop a worker, so only then is + * there a publication to coordinate. */ + if (cbm_index_policy_worker_enabled(&handle->resource_policy) && + !worker_publish_fence_create(handle)) { + char error_text[CBM_SZ_32]; + (void)snprintf(error_text, sizeof(error_text), "%d", errno); + cbm_log_error("index.supervisor.artifact_create_failed", "artifact", "publish_fence", + "errno", error_text); + (void)cbm_unlink(handle->response_path); + (void)cbm_unlink(handle->log_path); + free(handle); + return -1; + } const char *argv[INDEX_WORKER_ARGV_CAP]; size_t argc = 0; @@ -964,6 +1163,10 @@ static int worker_start_internal(const char *args_json, size_t memory_budget_byt argv[argc++] = CBM_INDEX_WORKER_QUARANTINE_ARG; argv[argc++] = quarantine_file; } + if (handle->publish_fence_path[0]) { + argv[argc++] = CBM_INDEX_WORKER_PUBLISH_FENCE_ARG; + argv[argc++] = handle->publish_fence_path; + } argv[argc] = NULL; cbm_proc_opts_t options = {0}; @@ -981,6 +1184,7 @@ static int worker_start_internal(const char *args_json, size_t memory_budget_byt if (cbm_subprocess_spawn(&options, &handle->process) != 0) { (void)cbm_unlink(handle->response_path); (void)cbm_unlink(handle->log_path); + worker_publish_fence_remove(handle); free(handle); cbm_log_error("index.supervisor.spawn_failed", "action", "fail_closed"); return -1; @@ -1089,6 +1293,12 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, handle->result.forced = process_result->forced; handle->result.tree_quiesced = process_result->tree_quiesced; handle->result.supervision_failed = process_result->supervision_failed; + handle->result.publication_committed = worker_publication_committed_at_terminal(handle); + if (handle->result.tree_quiesced) { + /* An unquiesced tree keeps the fence as it is: removing a claim the + * supervisor holds would let a surviving member publish later. */ + worker_publish_fence_remove(handle); + } if (worker_result_succeeded(&handle->result)) { worker_response_read_status_t response_status; handle->result.response = slurp_worker_response(handle->response_path, &response_status); diff --git a/src/mcp/index_supervisor.h b/src/mcp/index_supervisor.h index 2cb244a7ee..359a3e25e9 100644 --- a/src/mcp/index_supervisor.h +++ b/src/mcp/index_supervisor.h @@ -42,6 +42,7 @@ void cbm_index_set_worker_role(bool is_worker, const char *response_out); #define CBM_INDEX_WORKER_QUARANTINE_ARG "--index-worker-quarantine" #define CBM_INDEX_WORKER_MEMORY_BUDGET_ARG "--index-worker-memory-budget-bytes" #define CBM_INDEX_WORKER_BUILD_ARG "--index-worker-build" +#define CBM_INDEX_WORKER_PUBLISH_FENCE_ARG "--index-worker-publish-fence" #define CBM_INDEX_WORKER_BUILD_FINGERPRINT_LENGTH 64U #define CBM_INDEX_WORKER_BUILD_FINGERPRINT_SIZE 65U void cbm_index_set_worker_role_options(bool is_worker, const char *response_out, bool single_thread, @@ -51,6 +52,32 @@ bool cbm_index_worker_active(void); const char *cbm_index_worker_response_out(void); size_t cbm_index_worker_memory_budget_bytes(void); +/* Publication fence. A worker resource limit (index_max_rss_mb, + * index_max_duration_seconds) is enforced by the supervisor, while only the + * worker replaces the serving database. When such a limit is enabled, every + * worker attempt gets a fence file path, and publication and resource + * termination both CLAIM it by creating that file exclusively, so exactly one + * of them can win: + * - the worker claims immediately before it replaces the serving database; + * losing means the supervisor already decided to stop it for a limit, so + * the run aborts before publication and the previous index stays intact; + * - the supervisor claims before it terminates the worker for a limit; + * losing means the worker already committed to publishing, so the limit is + * no longer enforced for that attempt and the attempt runs to completion. + * A reported resource failure therefore never follows a publication by the + * same attempt. Without an enabled worker limit there is no fence and nothing + * changes. The supervisor passes the path on the worker argv; the worker entry + * installs it here (NULL clears it). */ +void cbm_index_set_worker_publish_fence(const char *fence_file); +const char *cbm_index_worker_publish_fence(void); + +/* Worker side: claim publication. True when publishing may proceed: no fence + * is installed, or this call created the fence file. False when the + * supervisor already claimed it (a resource limit stopped this attempt) or the + * claim could not be made; the caller must then leave the serving database + * untouched. Idempotent after a successful claim. */ +bool cbm_index_worker_claim_publication(void); + /* Event name of the worker log's startup header. Shared with the tests so the * contract has exactly one spelling. */ #define CBM_INDEX_WORKER_LOG_START_EVENT "index.worker.start" @@ -96,6 +123,7 @@ typedef struct { const char *marker_file; const char *quarantine_file; size_t memory_budget_bytes; + const char *publish_fence_file; } cbm_index_worker_invocation_t; typedef enum { @@ -160,6 +188,12 @@ typedef struct { bool response_rejected; /* clean worker exceeded the bounded response protocol */ cbm_index_resource_violation_t resource_violation; bool resource_probe_failed; + /* The worker claimed the publication fence: it committed to replacing the + * serving database (whether or not it got that far). Also true when the + * fence state could not be read, because a report may only claim the + * previous index was preserved when it can prove it. Always false without + * a fence (no worker limit enabled) and for an attempt stopped by a limit. */ + bool publication_committed; char *response; /* worker result only after a contained, uncancelled CLEAN exit; * borrowed for async polls, caller-owned from the sync wrapper */ } cbm_index_worker_result_t; @@ -283,6 +317,9 @@ void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_cl cbm_index_supervisor_rss_fn rss_fn, void *context); void cbm_index_supervisor_reset_resource_hooks_for_testing(void); +/* Borrowed fence path of an attempt; NULL when it has no fence. */ +const char *cbm_index_worker_publish_fence_path_for_testing( + const cbm_index_worker_handle_t *handle); #endif #endif /* CBM_INDEX_SUPERVISOR_H */ diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index f23a00e5cd..3dc5e72ee4 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -11277,16 +11277,50 @@ static char *build_worker_unsafe_terminal_response(const char *args, cbm_proc_ou return index_worker_json_result(doc, repo_path, NULL, NULL); } -char *cbm_mcp_index_worker_resource_response(const char *args, - const cbm_index_worker_result_t *worker_result) { +/* Whether the project named by internal worker args has a servable index now. */ +static bool index_args_serving_index_servable(const char *args) { char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); char *name_override = cbm_mcp_get_string_arg(args, "name"); char *project_name = cbm_project_name_from_path(name_override && name_override[0] ? name_override : repo_path); - char db_path[CBM_SZ_1K] = {0}; + bool servable = false; if (project_name) { + char db_path[CBM_SZ_1K] = {0}; project_db_path(project_name, db_path, sizeof(db_path)); + servable = project_db_is_servable(project_name, db_path); } + free(repo_path); + free(name_override); + free(project_name); + return servable; +} + +void cbm_mcp_index_request_facts_begin(const char *args, cbm_mcp_index_request_facts_t *facts) { + if (!facts) { + return; + } + facts->serving_index_existed = index_args_serving_index_servable(args); + facts->publication_committed = false; +} + +void cbm_mcp_index_request_facts_record(cbm_mcp_index_request_facts_t *facts, + const cbm_index_worker_result_t *worker_result) { + if (facts && worker_result && worker_result->publication_committed) { + facts->publication_committed = true; + } +} + +char *cbm_mcp_index_worker_resource_response(const char *args, + const cbm_index_worker_result_t *worker_result, + const cbm_mcp_index_request_facts_t *facts) { + char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); + /* The publication fence makes a limit-stopped attempt and a publication by + * that attempt mutually exclusive; an EARLIER attempt of the request may + * still have published (it crashed after committing), and an index that + * appeared during the request (artifact bootstrap) was not preserved. */ + bool serving_index_preserved = + facts && facts->serving_index_existed && !facts->publication_committed && + !worker_result->publication_committed && index_args_serving_index_servable(args); const cbm_index_resource_violation_t *violation = &worker_result->resource_violation; const char *config_key = cbm_index_resource_config_key(violation->resource); @@ -11311,13 +11345,12 @@ char *cbm_mcp_index_worker_resource_response(const char *args, yyjson_mut_obj_add_str(doc, root, "unit", cbm_index_resource_unit(violation->resource)); } yyjson_mut_obj_add_bool(doc, root, "retryable", true); - yyjson_mut_obj_add_bool(doc, root, "serving_index_preserved", - project_name && project_db_is_servable(project_name, db_path)); + yyjson_mut_obj_add_bool(doc, root, "serving_index_preserved", serving_index_preserved); yyjson_mut_obj_add_strcpy(doc, root, "message", message); if (repo_path) { yyjson_mut_obj_add_strcpy(doc, root, "repo_path", repo_path); } - return index_worker_json_result(doc, repo_path, name_override, project_name); + return index_worker_json_result(doc, repo_path, NULL, NULL); } /* Drop the cached store so the next query reopens whatever the worker wrote (each @@ -11618,6 +11651,12 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, const cbm_index_resource_policy_t *resource_policy) { invalidate_cached_store(srv); uint64_t request_started_ms = cbm_index_worker_now_ms(); + /* Only an enabled worker limit can produce a worker resource failure, the + * one response these facts are reported in. */ + cbm_mcp_index_request_facts_t facts = {0}; + if (cbm_index_policy_worker_enabled(resource_policy)) { + cbm_mcp_index_request_facts_begin(args, &facts); + } /* First attempt: normal parallel run. */ cbm_index_worker_result_t wr; @@ -11627,6 +11666,7 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, request_started_ms, &wr); cbm_mcp_supervised_result_disposition_t disposition = cbm_mcp_supervised_result_disposition(rc, &wr); + cbm_mcp_index_request_facts_record(&facts, &wr); if (wr.response_missing) { /* #1300: a clean exit without a response is a named failure. */ @@ -11649,7 +11689,7 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, return failure; } if (disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { - char *failure = cbm_mcp_index_worker_resource_response(args, &wr); + char *failure = cbm_mcp_index_worker_resource_response(args, &wr, &facts); cbm_index_worker_result_free(&wr); invalidate_cached_store(srv); return failure; @@ -11709,6 +11749,7 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, srv ? &srv->pipeline_cancel_requested : NULL, request_started_ms, &wr2); cbm_mcp_supervised_result_disposition_t recovery_disposition = cbm_mcp_supervised_result_disposition(rc2, &wr2); + cbm_mcp_index_request_facts_record(&facts, &wr2); if (wr2.response_missing) { /* #1300: a recovery run that exits cleanly without a response is * reported as such, not as a failed recovery start. */ @@ -11732,7 +11773,7 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, break; } if (recovery_disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { - resp = cbm_mcp_index_worker_resource_response(args, &wr2); + resp = cbm_mcp_index_worker_resource_response(args, &wr2, &facts); cbm_index_worker_result_free(&wr2); break; } @@ -11836,6 +11877,7 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, srv ? &srv->pipeline_cancel_requested : NULL, request_started_ms, &wrp); cbm_mcp_supervised_result_disposition_t partial_disposition = cbm_mcp_supervised_result_disposition(rcp, &wrp); + cbm_mcp_index_request_facts_record(&facts, &wrp); if (partial_disposition == CBM_MCP_SUPERVISED_RESULT_SUCCESS) { resp = wrp.response; /* transfer ownership to caller */ wrp.response = NULL; @@ -11848,7 +11890,7 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, unsafe_terminal = true; terminal_cancelled = wrp.cancellation_requested; } else if (partial_disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { - resp = cbm_mcp_index_worker_resource_response(args, &wrp); + resp = cbm_mcp_index_worker_resource_response(args, &wrp, &facts); } cbm_index_worker_result_free(&wrp); } @@ -12293,6 +12335,11 @@ static char *handle_index_repository_status(cbm_mcp_server_t *srv, const char *a return result ? result : cbm_mcp_text_result("index job status is unavailable", true); } +static bool index_worker_publish_gate(void *context) { + (void)context; + return cbm_index_worker_claim_publication(); +} + static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { bool async_mode = false; bool status_mode = false; @@ -12492,6 +12539,11 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { free(name_override); cbm_pipeline_set_persistence(p, persistence); cbm_pipeline_set_resource_policy(p, &resource_policy); + if (cbm_index_worker_publish_fence()) { + /* Supervised worker with a worker limit: claim publication from the + * supervisor before the serving database is replaced. */ + cbm_pipeline_set_publish_gate(p, index_worker_publish_gate, NULL); + } index_doc_candidates_apply(p, doc_candidates); char *project_name = heap_strdup(cbm_pipeline_project_name(p)); diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index 3ca7c6ee9f..59e769dd95 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -62,8 +62,27 @@ bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, const cbm_index_resource_policy_t *policy); bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resource_policy_t *policy, char *error, size_t error_size); + +/* Request-scoped facts a worker resource failure is reported against. One + * supervised request can run several worker attempts (crash/hang recovery), and + * serving_index_preserved describes the request, not only its last attempt: + * it is true only when a servable index served the project when the request + * began AND no attempt of the request committed to replacing it (the + * publication fence, index_supervisor.h) AND that index is still servable. */ +typedef struct { + bool serving_index_existed; + bool publication_committed; +} cbm_mcp_index_request_facts_t; + +/* Capture the facts at the start of a request, before its first worker. */ +void cbm_mcp_index_request_facts_begin(const char *args, cbm_mcp_index_request_facts_t *facts); +/* Fold one finished attempt into the request facts. */ +void cbm_mcp_index_request_facts_record(cbm_mcp_index_request_facts_t *facts, + const cbm_index_worker_result_t *worker_result); +/* NULL facts claim nothing: serving_index_preserved is then false. */ char *cbm_mcp_index_worker_resource_response(const char *args, - const cbm_index_worker_result_t *worker_result); + const cbm_index_worker_result_t *worker_result, + const cbm_mcp_index_request_facts_t *facts); /* Encode a doc_candidates setting the user set to a boolean on an internal * worker request (the worker reads no config). Unset or unparseable adds no diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index e14bfb7f76..f1965ad4ba 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -303,6 +303,9 @@ struct cbm_pipeline { * per pipeline so concurrent test/process activity cannot cross-trigger. */ void (*before_publish_hook)(cbm_pipeline_t *, const char *, void *); void *before_publish_hook_ctx; + /* Production publication gate (cbm_pipeline_set_publish_gate). */ + cbm_pipeline_publish_gate_fn publish_gate; + void *publish_gate_ctx; /* Test-only: sees the complete graph buffer before the dump releases its indexes. */ void (*before_dump_hook)(const cbm_gbuf_t *, void *); void *before_dump_hook_ctx; @@ -897,6 +900,14 @@ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled) { } } +void cbm_pipeline_set_publish_gate(cbm_pipeline_t *p, cbm_pipeline_publish_gate_fn gate, + void *context) { + if (p) { + p->publish_gate = gate; + p->publish_gate_ctx = gate ? context : NULL; + } +} + void cbm_pipeline_set_doc_candidates_enabled(cbm_pipeline_t *p, bool enabled) { if (p) { p->doc_candidates_choice = @@ -4763,6 +4774,17 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { return CBM_PIPELINE_ABORT_PRESERVE_DB; } + /* Last point at which the serving database is still untouched: the + * destination preparation below may already move it aside. */ + if (p->publish_gate && !p->publish_gate(p->publish_gate_ctx)) { + cbm_log_warn("pipeline.publish_refused", "path", final_path); + cleanup_staging_db(staging_path); + free(staging_path); + free(final_path); + pipeline_release_test_config(p); + return CBM_PIPELINE_ABORT_PRESERVE_DB; + } + cbm_replacement_prepare_t prepared = {0}; int prepare_rc = prepare_publish_destination(final_path, final_existed, backup_succeeded, &prepared); diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 072674d8d4..bbc856b392 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -56,6 +56,17 @@ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled); /* Apply a validated discovery resource policy. The value is copied. */ void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, const cbm_index_resource_policy_t *policy); +/* Publication gate. cbm_pipeline_run calls it once, after the last + * pre-publication check and immediately before the serving database is + * touched (never for a live no-op, which replaces nothing). Returning false + * aborts the run with CBM_PIPELINE_ABORT_PRESERVE_DB and the previous + * generation stays exactly as it was. A supervised index worker uses it to + * claim publication from its supervisor (see the publication fence in + * mcp/index_supervisor.h). NULL removes the gate. */ +typedef bool (*cbm_pipeline_publish_gate_fn)(void *context); +void cbm_pipeline_set_publish_gate(cbm_pipeline_t *p, cbm_pipeline_publish_gate_fn gate, + void *context); + /* Doc -> code candidates (the doc_candidates setting). Never set, the index * mode decides: on for full, off for moderate and fast. The setter makes an * explicit choice that wins in every mode. Off skips scoring documentation diff --git a/tests/test_index_supervisor.c b/tests/test_index_supervisor.c index bda5032384..f3218fdb14 100644 --- a/tests/test_index_supervisor.c +++ b/tests/test_index_supervisor.c @@ -239,18 +239,59 @@ TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar) { "/tmp/m", CBM_INDEX_WORKER_QUARANTINE_ARG, "/tmp/q", + CBM_INDEX_WORKER_PUBLISH_FENCE_ARG, + "/tmp/f", NULL, }; cbm_index_worker_invocation_t invocation; - ASSERT_EQ(cbm_index_worker_parse_process_argv(16, valid, &invocation), + ASSERT_EQ(cbm_index_worker_parse_process_argv(18, valid, &invocation), CBM_INDEX_WORKER_ARGV_VALID); - ASSERT_EQ(cbm_daemon_process_role(16, valid), CBM_DAEMON_PROCESS_WORKER); + ASSERT_EQ(cbm_daemon_process_role(18, valid), CBM_DAEMON_PROCESS_WORKER); ASSERT_STR_EQ(invocation.args_json, valid[6]); ASSERT_STR_EQ(invocation.response_out, valid[8]); ASSERT_EQ(invocation.memory_budget_bytes, 1024); ASSERT_TRUE(invocation.single_thread); ASSERT_STR_EQ(invocation.marker_file, valid[13]); ASSERT_STR_EQ(invocation.quarantine_file, valid[15]); + ASSERT_STR_EQ(invocation.publish_fence_file, valid[17]); + /* Without a worker limit there is no fence: the shorter grammar stays valid. */ + ASSERT_EQ(cbm_index_worker_parse_process_argv(16, valid, &invocation), + CBM_INDEX_WORKER_ARGV_VALID); + ASSERT_EQ(cbm_daemon_process_role(16, valid), CBM_DAEMON_PROCESS_WORKER); + ASSERT_NULL(invocation.publish_fence_file); + + char *fence_before_quarantine[] = {"test-runner", + "cli", + "--index-worker", + CBM_INDEX_WORKER_BUILD_ARG, + (char *)captured, + "index_repository", + "{}", + "--response-out", + "/tmp/r", + CBM_INDEX_WORKER_PUBLISH_FENCE_ARG, + "/tmp/f", + CBM_INDEX_WORKER_QUARANTINE_ARG, + "/tmp/q", + NULL}; + char *empty_fence[] = {"test-runner", + "cli", + "--index-worker", + CBM_INDEX_WORKER_BUILD_ARG, + (char *)captured, + "index_repository", + "{}", + "--response-out", + "/tmp/r", + CBM_INDEX_WORKER_PUBLISH_FENCE_ARG, + "", + NULL}; + ASSERT_EQ(cbm_index_worker_parse_process_argv(13, fence_before_quarantine, &invocation), + CBM_INDEX_WORKER_ARGV_INVALID); + ASSERT_EQ(cbm_daemon_process_role(13, fence_before_quarantine), CBM_DAEMON_PROCESS_INVALID); + ASSERT_EQ(cbm_index_worker_parse_process_argv(11, empty_fence, &invocation), + CBM_INDEX_WORKER_ARGV_INVALID); + ASSERT_EQ(cbm_daemon_process_role(11, empty_fence), CBM_DAEMON_PROCESS_INVALID); char *missing_build[] = {"test-runner", "cli", "--index-worker", "index_repository", "{}", "--response-out", @@ -1071,12 +1112,15 @@ TEST(index_supervisor_disabled_limits_do_not_probe) { cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"clean\"}", 0, &policy, false, NULL, NULL, 0, &handle); + /* Opt-in: no worker limit, no publication fence, no claim. */ + bool fenced = !handle || cbm_index_worker_publish_fence_path_for_testing(handle) != NULL; const cbm_index_worker_result_t *result = NULL; bool terminal = handle && index_supervisor_test_poll_terminal( handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); cbm_index_resource_t resource = result ? result->resource_violation.resource : CBM_INDEX_RESOURCE_RSS_BYTES; bool probe_failed = result && result->resource_probe_failed; + bool committed = !result || result->publication_committed; if (terminal) { cbm_index_worker_destroy(handle); } else { @@ -1090,6 +1134,8 @@ TEST(index_supervisor_disabled_limits_do_not_probe) { ASSERT_EQ(fake.clock_calls, 0); ASSERT_EQ(resource, CBM_INDEX_RESOURCE_NONE); ASSERT_FALSE(probe_failed); + ASSERT_FALSE(fenced); + ASSERT_FALSE(committed); PASS(); } @@ -1352,6 +1398,179 @@ TEST(index_supervisor_three_failed_rss_probes_fail_closed) { PASS(); } +/* Publication fence (index_supervisor.h). The hang-tree probe reports the fence + * path it received on its argv in its marker, and the test plays the worker's + * publication claim through the production entry point with that path. */ +typedef struct { + char directory[INDEX_SUPERVISOR_TEST_PATH_CAP]; + char marker[INDEX_SUPERVISOR_TEST_PATH_CAP]; + char fence[INDEX_SUPERVISOR_TEST_PATH_CAP]; +} index_supervisor_fence_fixture_t; + +static bool index_supervisor_fence_fixture_init(index_supervisor_fence_fixture_t *fixture) { + memset(fixture, 0, sizeof(*fixture)); + (void)snprintf(fixture->directory, sizeof(fixture->directory), "%s/cbm-index-fence-XXXXXX", + cbm_tmpdir()); + if (!cbm_mkdtemp(fixture->directory)) { + return false; + } + (void)snprintf(fixture->marker, sizeof(fixture->marker), "%s/marker", fixture->directory); + return true; +} + +/* True once the worker started with exactly this attempt's fence. */ +static bool index_supervisor_fence_fixture_started(index_supervisor_fence_fixture_t *fixture, + cbm_index_worker_handle_t *handle) { + const char *fence = handle ? cbm_index_worker_publish_fence_path_for_testing(handle) : NULL; + if (!fence) { + return false; + } + (void)snprintf(fixture->fence, sizeof(fixture->fence), "%s", fence); + char expected[INDEX_SUPERVISOR_TEST_PATH_CAP + 16]; + (void)snprintf(expected, sizeof(expected), "fence=%s\n", fixture->fence); + return index_supervisor_test_wait_file_text(fixture->marker, expected, + INDEX_SUPERVISOR_TEST_READY_MS); +} + +static bool index_supervisor_fence_fixture_worker_claims( + index_supervisor_fence_fixture_t *fixture) { + cbm_index_set_worker_publish_fence(fixture->fence); + bool claimed = cbm_index_worker_claim_publication(); + cbm_index_set_worker_publish_fence(NULL); + return claimed; +} + +static void index_supervisor_fence_fixture_cleanup(index_supervisor_fence_fixture_t *fixture) { + (void)cbm_unlink(fixture->marker); + (void)cbm_rmdir(fixture->directory); +} + +static int g_index_supervisor_after_publication_logs; + +static void index_supervisor_test_after_publication_sink(const char *line) { + if (line && strstr(line, "index.supervisor.resource_limit_after_publication")) { + g_index_supervisor_after_publication_logs++; + } +} + +TEST(index_supervisor_limit_claims_publication_before_terminating) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_EMPTY, + }; + index_supervisor_fence_fixture_t fixture; + bool fixture_ready = index_supervisor_fence_fixture_init(&fixture); + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_charged_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = + fixture_ready + ? cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, fixture.marker, NULL, 100, &handle) + : -1; + bool started = start_rc == 0 && index_supervisor_fence_fixture_started(&fixture, handle); + /* The limit is crossed while the worker has not claimed publication: the + * supervisor claims the fence in this poll, before it stops the worker. */ + fake.now_ms = 1101; + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t decided = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + /* Whatever the worker does next, it can no longer publish. */ + bool worker_barred = started && !index_supervisor_fence_fixture_worker_claims(&fixture); + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool limited = terminal && result && + result->resource_violation.resource == CBM_INDEX_RESOURCE_DURATION_MS && + result->resource_violation.observed == 1001 && !result->publication_committed && + !result->cancellation_requested && result->tree_quiesced; + bool fence_removed = terminal && fixture.fence[0] && cbm_file_size(fixture.fence) < 0; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + if (!started) { + index_supervisor_test_dump("marker", fixture.marker); + } + index_supervisor_fence_fixture_cleanup(&fixture); + + ASSERT_TRUE(fixture_ready); + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(started); + ASSERT_EQ(decided, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(worker_barred); + ASSERT_TRUE(limited); + ASSERT_TRUE(fence_removed); + PASS(); +} + +TEST(index_supervisor_limit_after_worker_publication_claim_is_not_enforced) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_EMPTY, + }; + index_supervisor_fence_fixture_t fixture; + bool fixture_ready = index_supervisor_fence_fixture_init(&fixture); + CBMLogLevel saved_level = cbm_log_get_level(); + cbm_log_set_level(CBM_LOG_INFO); + g_index_supervisor_after_publication_logs = 0; + cbm_log_set_sink_ex(index_supervisor_test_after_publication_sink, CBM_LOG_SINK_TEE); + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_charged_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = + fixture_ready + ? cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, fixture.marker, NULL, 100, &handle) + : -1; + bool started = start_rc == 0 && index_supervisor_fence_fixture_started(&fixture, handle); + /* The worker commits to publishing first; only then is the limit crossed. */ + bool worker_claimed = started && index_supervisor_fence_fixture_worker_claims(&fixture); + fake.now_ms = 1101; + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t decided = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + int clock_calls_at_decision = fake.clock_calls; + cbm_index_worker_poll_t after = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + bool disarmed = fake.clock_calls == clock_calls_at_decision; + /* The attempt runs on; the test ends it with an ordinary cancellation. */ + bool cancel_accepted = handle && cbm_index_worker_request_cancel(handle); + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool committed = terminal && result && result->cancellation_requested && + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed && result->publication_committed && + result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + cbm_log_set_sink(NULL); + cbm_log_set_level(saved_level); + if (!started) { + index_supervisor_test_dump("marker", fixture.marker); + } + index_supervisor_fence_fixture_cleanup(&fixture); + + ASSERT_TRUE(fixture_ready); + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(started); + ASSERT_TRUE(worker_claimed); + ASSERT_EQ(decided, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(after, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(disarmed); + ASSERT_EQ(g_index_supervisor_after_publication_logs, 1); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(committed); + PASS(); +} + SUITE(index_supervisor) { RUN_TEST(index_supervisor_job_memory_limit_has_floor_headroom_and_no_overflow); RUN_TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar); @@ -1370,4 +1589,6 @@ SUITE(index_supervisor) { RUN_TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled); RUN_TEST(index_supervisor_cancel_precedes_resource_probe); RUN_TEST(index_supervisor_three_failed_rss_probes_fail_closed); + RUN_TEST(index_supervisor_limit_claims_publication_before_terminating); + RUN_TEST(index_supervisor_limit_after_worker_publication_claim_is_not_enforced); } diff --git a/tests/test_main.c b/tests/test_main.c index 23b2621ee7..652893ce11 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -455,10 +455,12 @@ static void tf_index_worker_probe(const char *args_json, const char *response_ou FILE *ready = marker ? cbm_fopen(marker, "wb") : NULL; if (ready) { (void)fprintf( - ready, "single=%s\nmarker=%s\nquarantine=%s\nbudget=%zu\ndescendant=%ld\n", + ready, + "single=%s\nmarker=%s\nquarantine=%s\nbudget=%zu\ndescendant=%ld\nfence=%s\n", getenv("CBM_INDEX_SINGLE_THREAD") ? getenv("CBM_INDEX_SINGLE_THREAD") : "", marker, getenv("CBM_INDEX_QUARANTINE_FILE") ? getenv("CBM_INDEX_QUARANTINE_FILE") : "", - cbm_mem_budget(), descendant); + cbm_mem_budget(), descendant, + cbm_index_worker_publish_fence() ? cbm_index_worker_publish_fence() : ""); (void)fclose(ready); } (void)fprintf(stderr, "async worker hang-tree probe\n"); @@ -509,6 +511,7 @@ static int tf_maybe_run_index_worker(int argc, char **argv) { cbm_index_set_worker_role_options(true, invocation.response_out, invocation.single_thread, invocation.marker_file, invocation.quarantine_file, invocation.memory_budget_bytes); + cbm_index_set_worker_publish_fence(invocation.publish_fence_file); cbm_mem_init_with_cap(0.5, invocation.memory_budget_bytes); cbm_log_init_for_process(false, true); tf_index_worker_probe(invocation.args_json, invocation.response_out); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 3bfcb018ff..7b4c69d910 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -21677,6 +21677,289 @@ TEST(index_worker_resource_limit_is_trusted_structured_and_not_retried) { #endif } +#ifndef _WIN32 +#include + +/* A worker limit must never be reported after the same attempt published. + * The real worker replaces the serving database and only then exports the + * persistence artifact. A FIFO planted at the artifact's temp path holds it + * inside that export, after the rename, for as long as nobody reads the FIFO, + * and the duration clock crosses its limit only once the serving database has + * been replaced. The limit is therefore observed in the post-publication + * window by construction, never by timing: + * - uncoordinated, the supervisor stops the worker there and answers + * resource_limit_exceeded with serving_index_preserved=true although the + * previous generation is already gone; + * - with the publication fence the worker claimed publication before the + * rename, so the supervisor leaves the limit unenforced and logs that. + * Only that log line releases the FIFO; the worker then finishes and + * reports the published index. The artifact path is a directory, so the + * export itself fails afterwards, which the #1665 response names. */ +typedef struct { + char live_db[CBM_SZ_1K]; + ino_t previous_inode; + int over_limit_calls; +} index_publication_race_clock_t; + +static char g_publication_race_fifo[CBM_SZ_1K]; +static bool g_publication_race_released; + +static uint64_t index_publication_race_clock(void *context) { + index_publication_race_clock_t *clock = context; + struct stat status; + if (stat(clock->live_db, &status) != 0 || status.st_ino == clock->previous_inode) { + return 100; + } + clock->over_limit_calls++; + return 1101; +} + +static void index_publication_race_sink(const char *line) { + if (g_publication_race_released || !line || + !strstr(line, "index.supervisor.resource_limit_after_publication")) { + return; + } + g_publication_race_released = true; + /* Blocks only until the held worker opens its end, then reads to EOF. */ + int fifo = open(g_publication_race_fifo, O_RDONLY); + if (fifo >= 0) { + char buffer[CBM_SZ_4K]; + ssize_t got = 0; + do { + got = read(fifo, buffer, sizeof(buffer)); + } while (got > 0); + (void)close(fifo); + } +} + +enum { + IDXPUB_OK = 0, + IDXPUB_SETUP = 81, + IDXPUB_FIRST_INDEX = 82, + IDXPUB_HOLD_SETUP = 83, + IDXPUB_NO_RESPONSE = 84, + IDXPUB_NOT_REPLACED = 85, + IDXPUB_NOT_CROSSED = 86, + IDXPUB_LIMIT_AFTER_PUBLICATION = 87, + IDXPUB_NOT_RELEASED = 88, + IDXPUB_NOT_REPORTED_PUBLISHED = 89, +}; + +static int index_publication_race_first_generation(cbm_mcp_server_t *server, const char *repo, + const char *cache, + index_publication_race_clock_t *clock) { + char args[CBM_SZ_2K]; + (void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\",\"mode\":\"fast\"}", repo); + char *response = cbm_mcp_handle_tool(server, "index_repository", args); + char *text = extract_text_content(response); + yyjson_doc *doc = text ? yyjson_read(text, strlen(text), 0) : NULL; + yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; + const char *status = yyjson_get_str(yyjson_obj_get(root, "status")); + const char *project = yyjson_get_str(yyjson_obj_get(root, "project")); + struct stat live; + bool indexed = status && strcmp(status, "indexed") == 0 && project && + snprintf(clock->live_db, sizeof(clock->live_db), "%s/%s.db", cache, project) < + (int)sizeof(clock->live_db) && + stat(clock->live_db, &live) == 0; + if (indexed) { + clock->previous_inode = live.st_ino; + } + yyjson_doc_free(doc); + free(text); + free(response); + return indexed ? IDXPUB_OK : IDXPUB_FIRST_INDEX; +} + +static int index_publication_race_check(const char *repo, const char *cache) { + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + cbm_unsetenv("CBM_INDEX_SUPERVISOR"); + cbm_index_supervisor_mark_host(); + cbm_config_t *config = cbm_config_open(cache); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + if (!config || !server) { + cbm_mcp_server_free(server); + cbm_config_close(config); + return IDXPUB_SETUP; + } + cbm_mcp_server_set_config(server, config); + index_publication_race_clock_t clock = {0}; + int result = index_publication_race_first_generation(server, repo, cache, &clock); + + /* A second generation to publish, the hold, and a 1 s duration limit. */ + char art_dir[CBM_SZ_1K]; + char art_path[CBM_SZ_1K]; + (void)snprintf(art_dir, sizeof(art_dir), "%s/.codebase-memory", repo); + (void)snprintf(art_path, sizeof(art_path), "%s/graph.db.zst", art_dir); + (void)snprintf(g_publication_race_fifo, sizeof(g_publication_race_fifo), "%s.tmp", art_path); + if (result == IDXPUB_OK && + (th_write_file(TH_PATH(repo, "extra.c"), "int extra(void) { return 1; }\n") != 0 || + mkdir(art_dir, 0700) != 0 || mkdir(art_path, 0700) != 0 || + mkfifo(g_publication_race_fifo, 0600) != 0 || + cbm_config_set(config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "1") != 0)) { + result = IDXPUB_HOLD_SETUP; + } + char *response = NULL; + if (result == IDXPUB_OK) { + char args[CBM_SZ_2K]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"mode\":\"fast\",\"persistence\":true}", repo); + cbm_log_set_level(CBM_LOG_WARN); + cbm_log_set_sink_ex(index_publication_race_sink, CBM_LOG_SINK_TEE); + cbm_index_supervisor_set_resource_hooks_for_testing(index_publication_race_clock, NULL, + &clock); + response = cbm_mcp_handle_tool(server, "index_repository", args); + cbm_index_supervisor_reset_resource_hooks_for_testing(); + cbm_log_set_sink(NULL); + } + struct stat live; + bool replaced = stat(clock.live_db, &live) == 0 && live.st_ino != clock.previous_inode; + if (result != IDXPUB_OK) { + /* setup failure already recorded */ + } else if (!response) { + result = IDXPUB_NO_RESPONSE; + } else if (!replaced) { + result = IDXPUB_NOT_REPLACED; + } else if (clock.over_limit_calls == 0) { + result = IDXPUB_NOT_CROSSED; + } else if (strstr(response, "resource_limit_exceeded")) { + result = IDXPUB_LIMIT_AFTER_PUBLICATION; + } else if (!g_publication_race_released) { + result = IDXPUB_NOT_RELEASED; + } else if (!strstr(response, "Index database was published")) { + result = IDXPUB_NOT_REPORTED_PUBLISHED; + } + if (result != IDXPUB_OK && response) { + printf(" second response: %.600s\n", response); + fflush(stdout); + } + free(response); + cbm_mcp_server_free(server); + cbm_config_close(config); + return result; +} +#endif + +TEST(index_worker_limit_never_reported_after_its_own_publication) { +#ifdef _WIN32 + SKIP_PLATFORM("fork + FIFO hold harness; the fence state machine is cross-platform and " + "covered by the index_supervisor suite"); +#else + /* th_mktempdir returns one static buffer: keep each path in its own copy. */ + const char *made = th_mktempdir("cbm_publication_race_repo"); + char *repo = made ? strdup(made) : NULL; + made = th_mktempdir("cbm_publication_race_cache"); + char *cache = made ? strdup(made) : NULL; + ASSERT_NOT_NULL(repo); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 0; }\n"), 0); + fflush(NULL); + pid_t pid = fork(); + if (pid == 0) { + /* Hang guard only: two supervised runs that each start an ASan worker. */ + alarm(180); + _exit(index_publication_race_check(repo, cache)); + } + int status = 0; + bool waited = waitpid(pid, &status, 0) == pid; + int exit_code = waited && WIFEXITED(status) ? WEXITSTATUS(status) : -1; + int signal_code = waited && WIFSIGNALED(status) ? WTERMSIG(status) : 0; + th_cleanup(repo); + th_cleanup(cache); + free(repo); + free(cache); + if (exit_code != IDXPUB_OK) { + printf(" publication race child exit=%d signal=%d\n", exit_code, signal_code); + } + ASSERT_TRUE(waited); + ASSERT_EQ(signal_code, 0); + ASSERT_EQ(exit_code, IDXPUB_OK); + PASS(); +#endif +} + +/* serving_index_preserved describes the request, not only its last attempt: + * an earlier attempt may have published before it crashed (recovery then runs + * another attempt), and an index that appeared during the request (artifact + * bootstrap) was not preserved. */ +TEST(index_worker_resource_response_reports_preservation_for_the_request) { + const char *made = th_mktempdir("cbm_request_facts_repo"); + char *repo = made ? strdup(made) : NULL; + made = th_mktempdir("cbm_request_facts_cache"); + char *cache = made ? strdup(made) : NULL; + ASSERT_NOT_NULL(repo); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 0; }\n"), 0); + const char *old_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache = old_cache ? strdup(old_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + char args[CBM_SZ_2K]; + (void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\",\"name\":\"RequestFactsFixture\"}", + repo); + + cbm_mcp_index_request_facts_t before_index = {0}; + cbm_mcp_index_request_facts_begin(args, &before_index); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + char *indexed = server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + cbm_mcp_server_free(server); + cbm_mcp_index_request_facts_t facts = {0}; + cbm_mcp_index_request_facts_begin(args, &facts); + + cbm_index_worker_result_t limited; + memset(&limited, 0, sizeof(limited)); + limited.outcome = CBM_PROC_KILLED; + limited.tree_quiesced = true; + limited.resource_violation = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_DURATION_MS, .observed = 1001, .limit = 1000}; + cbm_index_worker_result_t published_then_crashed; + memset(&published_then_crashed, 0, sizeof(published_then_crashed)); + published_then_crashed.outcome = CBM_PROC_CRASH; + published_then_crashed.publication_committed = true; + cbm_mcp_index_request_facts_t after_recovery = facts; + cbm_mcp_index_request_facts_record(&after_recovery, &limited); + bool limited_attempt_commits_nothing = !after_recovery.publication_committed; + cbm_mcp_index_request_facts_record(&after_recovery, &published_then_crashed); + + char *preserved = cbm_mcp_index_worker_resource_response(args, &limited, &facts); + char *earlier_published = + cbm_mcp_index_worker_resource_response(args, &limited, &after_recovery); + char *appeared = cbm_mcp_index_worker_resource_response(args, &limited, &before_index); + char *no_facts = cbm_mcp_index_worker_resource_response(args, &limited, NULL); + const char *kept = "\\\"serving_index_preserved\\\":true"; + const char *not_kept = "\\\"serving_index_preserved\\\":false"; + bool indexed_ok = indexed && strstr(indexed, "\\\"status\\\":\\\"indexed\\\""); + bool preserved_ok = preserved && strstr(preserved, kept); + bool earlier_ok = earlier_published && strstr(earlier_published, not_kept); + bool appeared_ok = appeared && strstr(appeared, not_kept); + bool no_facts_ok = no_facts && strstr(no_facts, not_kept); + + free(indexed); + free(preserved); + free(earlier_published); + free(appeared); + free(no_facts); + if (saved_cache) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache, 1); + } else { + cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache); + th_cleanup(repo); + th_cleanup(cache); + free(repo); + free(cache); + + ASSERT_FALSE(before_index.serving_index_existed); + ASSERT_TRUE(indexed_ok); + ASSERT_TRUE(facts.serving_index_existed); + ASSERT_FALSE(facts.publication_committed); + ASSERT_TRUE(limited_attempt_commits_nothing); + ASSERT_TRUE(after_recovery.publication_committed); + ASSERT_TRUE(preserved_ok); + ASSERT_TRUE(earlier_ok); + ASSERT_TRUE(appeared_ok); + ASSERT_TRUE(no_facts_ok); + PASS(); +} /* Child-side check: index a tiny fixture and verify it ran IN-PROCESS. * Distinct exit codes so the parent can report the exact failure mode. */ @@ -25238,6 +25521,8 @@ SUITE(mcp) { RUN_TEST(index_supervisor_start_failure_is_fail_closed_in_real_host); RUN_TEST(index_supervisor_clean_exit_without_response_response_issue1300); RUN_TEST(index_worker_resource_limit_is_trusted_structured_and_not_retried); + RUN_TEST(index_worker_limit_never_reported_after_its_own_publication); + RUN_TEST(index_worker_resource_response_reports_preservation_for_the_request); RUN_TEST(index_bg_paths_route_through_supervisor_issue832); RUN_TEST(sequential_service_edge_props_are_valid_json_issue898); RUN_TEST(index_second_inprocess_run_survives_issue773); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 2b80f51bdf..2441c205e2 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -21733,6 +21733,72 @@ TEST(cancelled_incremental_reindex_preserves_committed_db) { PASS(); } +typedef struct { + bool allow; + int calls; +} publish_gate_ctx_t; + +static bool publish_gate_decide(void *context) { + publish_gate_ctx_t *gate = context; + gate->calls++; + return gate->allow; +} + +/* The publication gate (a supervised worker's claim on its publication fence) + * is the last word before the serving database is touched: refused, the run + * stops with the previous generation's file in place; allowed, it publishes + * as before. Asked exactly once either way. */ +TEST(publish_gate_decides_before_the_committed_db_is_touched) { + if (setup_incremental_repo() != 0) { + FAIL("setup failed"); + } + cbm_pipeline_t *p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + char *project = strdup(cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + ASSERT_NOT_NULL(project); + struct stat baseline; + ASSERT_EQ(stat(g_incr_dbpath, &baseline), 0); + + ASSERT_TRUE(write_go_file(g_incr_tmpdir, "gate_candidate.go", + "package main\n\nfunc GateCandidate() int { return 7 }\n")); + publish_gate_ctx_t refused = {.allow = false}; + p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + cbm_pipeline_set_publish_gate(p, publish_gate_decide, &refused); + int refused_rc = cbm_pipeline_run(p); + cbm_pipeline_free(p); + struct stat after_refusal; + bool same_file = + stat(g_incr_dbpath, &after_refusal) == 0 && after_refusal.st_ino == baseline.st_ino; + cbm_store_t *live = cbm_store_open_path(g_incr_dbpath); + int refused_candidates = live ? count_nodes_named(live, project, "GateCandidate") : -1; + cbm_store_close(live); + + publish_gate_ctx_t allowed = {.allow = true}; + p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + cbm_pipeline_set_publish_gate(p, publish_gate_decide, &allowed); + int allowed_rc = cbm_pipeline_run(p); + cbm_pipeline_free(p); + live = cbm_store_open_path(g_incr_dbpath); + int allowed_candidates = live ? count_nodes_named(live, project, "GateCandidate") : -1; + cbm_store_close(live); + + free(project); + cleanup_incremental_repo(); + + ASSERT_EQ(refused.calls, 1); + ASSERT_EQ(refused_rc, CBM_PIPELINE_ABORT_PRESERVE_DB); + ASSERT_TRUE(same_file); + ASSERT_EQ(refused_candidates, 0); + ASSERT_EQ(allowed.calls, 1); + ASSERT_EQ(allowed_rc, 0); + ASSERT_GT(allowed_candidates, 0); + PASS(); +} + TEST(backup_failed_publish_failure_preserves_final_sidecars) { if (setup_incremental_repo() != 0) { FAIL("setup failed"); @@ -26421,6 +26487,7 @@ SUITE(pipeline) { RUN_TEST(incremental_new_file_added); RUN_TEST(cancelled_full_reindex_preserves_committed_db); RUN_TEST(cancelled_incremental_reindex_preserves_committed_db); + RUN_TEST(publish_gate_decides_before_the_committed_db_is_touched); RUN_TEST(backup_failed_publish_failure_preserves_final_sidecars); RUN_TEST(backup_failed_rename_failure_preserves_corrupt_main); #ifdef __linux__ From e8f0f4a2898b6d2b4468dd6cb984a39372e86ded Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 2 Nov 2026 18:23:18 +0100 Subject: [PATCH 22/27] fix(cli): accept the emptied hooks: {} mapping in the Hermes hook editor Removing the only Hermes pre_llm_call hook empties the hooks mapping, and the mapping editor writes an emptied section as hooks: {} - the same convention it uses for mcp_servers: {}. The hook sequence editor then refused that file for both upsert and removal, because it required every parent on the sequence path to be a block mapping header. On a config where our hook was the only entry under hooks, install -> uninstall -> install exited 1, and a repeated uninstall did too. An exact empty flow mapping on a parent of the sequence path is now read as that empty mapping, mirroring how the mapping-entry editor already treats section: {}. Removal is a byte-identical no-op, and upsert expands the header in place, keeping its comment, the line endings and every other byte, so a reinstall lands back on the first install byte for byte. The sequence key itself holding a mapping, any other flow value, and content nested under the inline mapping stay refused. Hooks written by the user and every foreign key are left untouched. Found while finishing #1862: its persisted hook opt-out runs this remover on every later install, so every Hermes user whose only hook was ours would have hit the refusal on each install after the first. Signed-off-by: Martin Vogel --- src/cli/config_yaml_edit.c | 85 +++++++++++++++++++-- tests/test_cli.c | 139 ++++++++++++++++++++++++++++++++++ tests/test_config_yaml_edit.c | 126 ++++++++++++++++++++++++++++++ 3 files changed, 345 insertions(+), 5 deletions(-) diff --git a/src/cli/config_yaml_edit.c b/src/cli/config_yaml_edit.c index a6f8ced105..09a079212f 100644 --- a/src/cli/config_yaml_edit.c +++ b/src/cli/config_yaml_edit.c @@ -2589,6 +2589,10 @@ typedef struct { bool identity_found; size_t identity_start; size_t identity_end; + /* The missing rest of the path begins under a `parent: {}` header. */ + bool parent_inline_empty; + size_t parent_line; + size_t parent_colon; } yaml_mapping_sequence_target_t; static void yaml_sequence_key_vec_free(yaml_sequence_key_vec_t *keys) { @@ -2962,6 +2966,37 @@ static int yaml_sequence_parse_block(const yaml_doc_t *doc, size_t begin_line, s return 0; } +/* `section: {}` is how the mapping-entry editor writes a section it emptied + * (yaml_build_last_mapping_entry_removal), and its upsert accepts it back. On + * a PARENT of the sequence path it is that same empty mapping, so the rest of + * the path is missing beneath it: removal is a no-op and upsert expands the + * header in place. The Hermes remover leaves `hooks: {}` whenever our hook was + * the only entry. The sequence key itself holding `{}`, any other flow value, + * and content nested under the inline mapping stay refused. */ +static int yaml_sequence_target_empty_parent(const yaml_doc_t *doc, size_t key_line, size_t colon, + size_t child_end, size_t next_depth, + size_t sequence_path_len, + yaml_mapping_sequence_target_t *target) { + bool inline_empty = false; + if (next_depth >= sequence_path_len || + yaml_tail_is_explicit_empty_mapping(doc->data, colon, doc->lines[key_line].text_end, + &inline_empty) != 0 || + !inline_empty) { + return YAML_ERROR; + } + for (size_t i = key_line + YAML_UNIT; i < child_end; i++) { + if (!doc->lines[i].blank && !doc->lines[i].comment) { + return YAML_ERROR; + } + } + target->missing_index = next_depth; + target->insert_offset = yaml_sequence_line_offset(doc, child_end); + target->parent_inline_empty = true; + target->parent_line = key_line; + target->parent_colon = colon; + return 0; +} + static int yaml_sequence_analyze(const yaml_doc_t *doc, const char *const *sequence_path, const size_t *path_lengths, size_t sequence_path_len, const char *identity_key, const char *identity_value, @@ -2993,10 +3028,11 @@ static int yaml_sequence_analyze(const yaml_doc_t *doc, const char *const *seque return 0; } const yaml_line_t *header = &doc->lines[key_line]; + size_t child_end = yaml_sequence_nested_end(doc, key_line, parent_end); if (!yaml_tail_is_empty(doc->data, colon, header->text_end)) { - return YAML_ERROR; + return yaml_sequence_target_empty_parent(doc, key_line, colon, child_end, + depth + YAML_UNIT, sequence_path_len, target); } - size_t child_end = yaml_sequence_nested_end(doc, key_line, parent_end); if (depth + YAML_UNIT == sequence_path_len) { target->sequence_found = true; target->missing_index = sequence_path_len; @@ -3095,13 +3131,52 @@ static int yaml_sequence_validate_canonical(const char *canonical_item, size_t c return result; } +/* Copies the document up to insert_offset with the `parent: {}` header turned + * into a block header `parent:`, exactly as + * yaml_build_inline_empty_mapping_section_add does for a mapping section: a + * trailing comment and the line ending are kept. */ +static int yaml_sequence_append_expanded_parent(yaml_buf_t *out, const yaml_doc_t *doc, + const yaml_mapping_sequence_target_t *target) { + if (target->parent_line >= doc->line_count) { + return YAML_ERROR; + } + const yaml_line_t *header = &doc->lines[target->parent_line]; + size_t comment = 0U; + if (target->insert_offset < header->end || + yaml_find_comment(doc->data, target->parent_colon + YAML_UNIT, header->text_end, + &comment) != 0 || + yaml_buf_append(out, doc->data, target->parent_colon + YAML_UNIT) != 0) { + return YAML_ERROR; + } + size_t kept_tail = header->text_end; + if (comment < header->text_end) { + if (yaml_buf_append_char(out, ' ') != 0) { + return YAML_ERROR; + } + kept_tail = comment; + } + if (yaml_buf_append(out, doc->data + kept_tail, header->end - kept_tail) != 0 || + yaml_buf_append(out, doc->data + header->end, target->insert_offset - header->end) != 0 || + (out->len > 0U && out->data[out->len - YAML_UNIT] != '\n' && + yaml_buf_append(out, doc->eol, doc->eol_len) != 0)) { + return YAML_ERROR; + } + return 0; +} + static int yaml_sequence_build_insert(yaml_buf_t *out, const yaml_doc_t *doc, const yaml_mapping_sequence_target_t *target, const char *const *sequence_path, const size_t *path_lengths, size_t sequence_path_len, const yaml_buf_t *rendered_item) { - if (target->insert_offset > doc->len || - yaml_buf_append(out, doc->data, target->insert_offset) != 0 || - yaml_append_separator_if_needed(out, doc, target->insert_offset) != 0) { + if (target->insert_offset > doc->len) { + return YAML_ERROR; + } + if (target->parent_inline_empty) { + if (yaml_sequence_append_expanded_parent(out, doc, target) != 0) { + return YAML_ERROR; + } + } else if (yaml_buf_append(out, doc->data, target->insert_offset) != 0 || + yaml_append_separator_if_needed(out, doc, target->insert_offset) != 0) { return YAML_ERROR; } if (!target->sequence_found) { diff --git a/tests/test_cli.c b/tests/test_cli.c index f716418ad5..c5417b4e22 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -9449,6 +9449,143 @@ TEST(cli_hermes_stable_shell_context_contract) { PASS(); } +/* A fake HOME holding only ~/.hermes/config.yaml, so install and uninstall + * reach exactly the Hermes editors. */ +typedef struct { + char tmpdir[256]; + char config_path[640]; + char binary_path[640]; + char *saved_home; + char *saved_path; + char *saved_hermes; +} cli_hermes_cycle_t; + +static bool cli_hermes_cycle_open(cli_hermes_cycle_t *cycle, const char *initial_config) { + memset(cycle, 0, sizeof(*cycle)); + snprintf(cycle->tmpdir, sizeof(cycle->tmpdir), "/tmp/cli-hermes-cycle-XXXXXX"); + if (!cbm_mkdtemp(cycle->tmpdir)) { + cycle->tmpdir[0] = '\0'; + return false; + } + cycle->saved_home = save_test_env("HOME"); + cycle->saved_path = save_test_env("PATH"); + cycle->saved_hermes = save_test_env("HERMES_HOME"); + cbm_setenv("HOME", cycle->tmpdir, 1); + cbm_setenv("PATH", cycle->tmpdir, 1); + cbm_unsetenv("HERMES_HOME"); + + char hermes_dir[512]; + snprintf(hermes_dir, sizeof(hermes_dir), "%s/.hermes", cycle->tmpdir); + snprintf(cycle->config_path, sizeof(cycle->config_path), "%s/config.yaml", hermes_dir); +#ifdef _WIN32 + snprintf(cycle->binary_path, sizeof(cycle->binary_path), + "%s/.local/bin/codebase-memory-mcp.exe", cycle->tmpdir); +#else + snprintf(cycle->binary_path, sizeof(cycle->binary_path), "%s/.local/bin/codebase-memory-mcp", + cycle->tmpdir); +#endif + test_mkdirp(hermes_dir); + return write_test_file(cycle->config_path, initial_config) == 0; +} + +static void cli_hermes_cycle_close(cli_hermes_cycle_t *cycle) { + if (!cycle->tmpdir[0]) { + return; + } + restore_test_env("HOME", cycle->saved_home); + restore_test_env("PATH", cycle->saved_path); + restore_test_env("HERMES_HOME", cycle->saved_hermes); + test_rmdir_r(cycle->tmpdir); + cycle->tmpdir[0] = '\0'; +} + +/* Found while finishing #1862: when our hook was the only entry under + * `hooks:`, uninstall empties that mapping and the mapping editor writes it as + * `hooks: {}` (the same convention as `mcp_servers: {}`). The hook sequence + * editor then refused the file, so install -> uninstall -> install exited 1, + * and so did a repeated uninstall. The explicit empty mapping must be accepted: + * a repeated uninstall changes nothing and the reinstall restores the first + * install byte for byte, with exactly one owned hook. */ +TEST(cli_hermes_hook_reinstalls_after_uninstall_empties_hooks) { + cli_hermes_cycle_t cycle; + if (!cli_hermes_cycle_open(&cycle, "theme: solarized\n")) { + cli_hermes_cycle_close(&cycle); + FAIL("Hermes fake-HOME setup failed"); + } + char *argv[] = {"uninstall", "--yes"}; + int install_rc = cbm_install_agent_configs(cycle.tmpdir, cycle.binary_path, false, false); + char *installed = read_test_file_alloc(cycle.config_path); + int uninstall_rc = cli_test_cmd_uninstall(2, argv); + char *emptied = read_test_file_alloc(cycle.config_path); + int repeat_rc = cli_test_cmd_uninstall(2, argv); + char *repeated = read_test_file_alloc(cycle.config_path); + int reinstall_rc = cbm_install_agent_configs(cycle.tmpdir, cycle.binary_path, false, false); + char *reinstalled = read_test_file_alloc(cycle.config_path); + cli_hermes_cycle_close(&cycle); + + bool installed_once = install_rc == 0 && installed && + test_count_substring(installed, "pre_llm_call:") == 1U && + test_count_substring(installed, "hook-augment") == 1U; + bool emptied_clean = uninstall_rc == 0 && emptied && strstr(emptied, "theme: solarized\n") && + test_count_substring(emptied, "hooks: {}\n") == 1U && + !strstr(emptied, "pre_llm_call") && !strstr(emptied, "hook-augment"); + bool repeat_unchanged = repeat_rc == 0 && emptied && repeated && strcmp(repeated, emptied) == 0; + bool reinstalled_once = + reinstall_rc == 0 && installed && reinstalled && strcmp(reinstalled, installed) == 0; + free(installed); + free(emptied); + free(repeated); + free(reinstalled); + if (!installed_once) + FAIL("Hermes install must write exactly one owned pre_llm_call hook"); + if (!emptied_clean) + FAIL("uninstall must remove the owned hook and leave only the empty hooks: {} mapping"); + if (!repeat_unchanged) + FAIL("a repeated uninstall must accept hooks: {} and leave the file unchanged"); + if (!reinstalled_once) + FAIL("reinstall must accept hooks: {} and restore exactly one owned hook"); + PASS(); +} + +/* The same cycle with a user's own hook in the shared pre_llm_call sequence: + * the foreign hook survives untouched and uninstall restores the original + * file byte for byte. */ +TEST(cli_hermes_hook_cycle_keeps_foreign_pre_llm_hook) { + static const char original[] = "theme: solarized\n" + "mcp_servers: {}\n" + "hooks:\n" + " pre_llm_call:\n" + " - command: \"/usr/bin/user-hermes-hook\"\n"; + cli_hermes_cycle_t cycle; + if (!cli_hermes_cycle_open(&cycle, original)) { + cli_hermes_cycle_close(&cycle); + FAIL("Hermes fake-HOME setup failed"); + } + char *argv[] = {"uninstall", "--yes"}; + int install_rc = cbm_install_agent_configs(cycle.tmpdir, cycle.binary_path, false, false); + char *installed = read_test_file_alloc(cycle.config_path); + int uninstall_rc = cli_test_cmd_uninstall(2, argv); + char *restored = read_test_file_alloc(cycle.config_path); + int reinstall_rc = cbm_install_agent_configs(cycle.tmpdir, cycle.binary_path, false, false); + char *reinstalled = read_test_file_alloc(cycle.config_path); + cli_hermes_cycle_close(&cycle); + + bool merged = install_rc == 0 && installed && + test_count_substring(installed, "/usr/bin/user-hermes-hook") == 1U && + test_count_substring(installed, "pre_llm_call:") == 1U && + test_count_substring(installed, "hook-augment") == 1U; + bool restored_exactly = uninstall_rc == 0 && restored && strcmp(restored, original) == 0; + bool reinstalled_once = + reinstall_rc == 0 && installed && reinstalled && strcmp(reinstalled, installed) == 0; + free(installed); + free(restored); + free(reinstalled); + if (!merged || !restored_exactly || !reinstalled_once) + FAIL("a user's own pre_llm_call hook must survive install, uninstall and reinstall, and " + "uninstall must restore the original file byte for byte"); + PASS(); +} + #ifndef _WIN32 TEST(cli_detected_agent_summary_includes_registry_clients) { char tmpdir[256]; @@ -20351,6 +20488,8 @@ SUITE(cli) { RUN_TEST(cli_junie_current_durable_context_contract); RUN_TEST(cli_rovo_installs_documented_global_memory); RUN_TEST(cli_hermes_stable_shell_context_contract); + RUN_TEST(cli_hermes_hook_reinstalls_after_uninstall_empties_hooks); + RUN_TEST(cli_hermes_hook_cycle_keeps_foreign_pre_llm_hook); #ifndef _WIN32 RUN_TEST(cli_detected_agent_summary_includes_registry_clients); #endif diff --git a/tests/test_config_yaml_edit.c b/tests/test_config_yaml_edit.c index 9efaa2e389..95bc89404b 100644 --- a/tests/test_config_yaml_edit.c +++ b/tests/test_config_yaml_edit.c @@ -1998,6 +1998,129 @@ TEST(config_yaml_edit_nested_sequence_removes_only_exact_canonical_item) { PASS(); } +/* Found while finishing #1862: cbm_yaml_remove_mapping_entry writes a section + * it empties as `section: {}`, and the Hermes remover empties `hooks:` whenever + * our hook was its only entry. The sequence editor refused that explicit empty + * parent mapping, so the next install (and a repeated removal) failed. The + * primitives' own round trip must land back on the first insert. */ +TEST(config_yaml_edit_nested_sequence_reinserts_under_emptied_parent) { + const char *initial = "theme: x\n"; + yaml_fixture_t fixture; + ASSERT_EQ(yaml_fixture_init(&fixture, initial), 0); + ASSERT_EQ(cbm_yaml_upsert_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, 2U, "id", + yaml_hook_identity, yaml_hook_canonical_item), + CBM_YAML_IDENTITY_EDIT_OK); + char *first = yaml_read_alloc(fixture.path); + ASSERT_NOT_NULL(first); + ASSERT_EQ(cbm_yaml_remove_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, 2U, "id", + yaml_hook_identity, yaml_hook_canonical_item), + CBM_YAML_IDENTITY_EDIT_OK); + ASSERT_EQ(cbm_yaml_remove_mapping_entry(fixture.path, "hooks", "pre_llm_call"), 0); + char *emptied = yaml_read_alloc(fixture.path); + ASSERT_NOT_NULL(emptied); + ASSERT_STR_EQ(emptied, "theme: x\nhooks: {}\n"); + free(emptied); + ASSERT_EQ(cbm_yaml_remove_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, 2U, "id", + yaml_hook_identity, yaml_hook_canonical_item), + CBM_YAML_IDENTITY_EDIT_OK); + ASSERT_EQ(cbm_yaml_upsert_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, 2U, "id", + yaml_hook_identity, yaml_hook_canonical_item), + CBM_YAML_IDENTITY_EDIT_OK); + char *again = yaml_read_alloc(fixture.path); + ASSERT_NOT_NULL(again); + ASSERT_STR_EQ(again, first); + free(again); + free(first); + th_cleanup(fixture.dir); + PASS(); +} + +/* An exact empty flow mapping on a PARENT of the sequence path is an empty + * mapping, the same `section: {}` the mapping-entry editor writes and accepts. + * Removal is a byte-identical no-op; upsert expands the header in place, + * keeping its comment, the line endings and every other byte. */ +TEST(config_yaml_edit_nested_sequence_expands_explicit_empty_parent_mapping) { + static const char item_lf[] = " pre_llm_call:\n" + " - id: \"codebase-memory-mcp\"\n" + " type: \"command\"\n" + " command: \"/opt/Codebase Memory/bin/cbm\"\n"; + static const char item_crlf[] = " pre_llm_call:\r\n" + " - id: \"codebase-memory-mcp\"\r\n" + " type: \"command\"\r\n" + " command: \"/opt/Codebase Memory/bin/cbm\"\r\n"; + struct { + const char *initial; + const char *header; + const char *item; + const char *tail; + } cases[] = { + {"theme: x\nhooks: {} # keep me\nafter: 1\n", "theme: x\nhooks: # keep me\n", item_lf, + "after: 1\n"}, + {"theme: x\r\nhooks: {}\r\nafter: 1\r\n", "theme: x\r\nhooks:\r\n", item_crlf, + "after: 1\r\n"}, + {"theme: x\nhooks: {}", "theme: x\nhooks:\n", item_lf, ""}, + {"hooks: {}\n# trailing note\n", "hooks:\n# trailing note\n", item_lf, ""}, + }; + for (size_t i = 0U; i < sizeof(cases) / sizeof(cases[0]); i++) { + char expected[1024]; + ASSERT(snprintf(expected, sizeof(expected), "%s%s%s", cases[i].header, cases[i].item, + cases[i].tail) > 0); + yaml_fixture_t fixture; + ASSERT_EQ(yaml_fixture_init(&fixture, cases[i].initial), 0); + ASSERT_EQ(cbm_yaml_remove_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, 2U, + "id", yaml_hook_identity, + yaml_hook_canonical_item), + CBM_YAML_IDENTITY_EDIT_OK); + char *unchanged = yaml_read_alloc(fixture.path); + ASSERT_NOT_NULL(unchanged); + ASSERT_STR_EQ(unchanged, cases[i].initial); + free(unchanged); + for (int pass = 0; pass < 2; pass++) { + ASSERT_EQ(cbm_yaml_upsert_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, + 2U, "id", yaml_hook_identity, + yaml_hook_canonical_item), + CBM_YAML_IDENTITY_EDIT_OK); + char *expanded = yaml_read_alloc(fixture.path); + ASSERT_NOT_NULL(expanded); + ASSERT_STR_EQ(expanded, expected); + free(expanded); + } + th_cleanup(fixture.dir); + } + PASS(); +} + +/* Only an exact `{}` on a parent with nothing nested beneath it is accepted. + * The sequence key itself holding a mapping, a non-empty flow mapping, a flow + * sequence where a mapping belongs, and content indented under an inline `{}` + * stay refused byte-identically by both upsert and removal. */ +TEST(config_yaml_edit_nested_sequence_still_refuses_non_empty_parent_flow) { + const char *cases[] = { + "hooks:\n pre_llm_call: {}\n", + "hooks: {pre_llm_call: []}\n", + "hooks: []\n", + "hooks: {}\n pre_llm_call:\n - id: \"codebase-memory-mcp\"\n", + }; + for (size_t i = 0U; i < sizeof(cases) / sizeof(cases[0]); i++) { + yaml_fixture_t fixture; + ASSERT_EQ(yaml_fixture_init(&fixture, cases[i]), 0); + ASSERT_EQ(cbm_yaml_upsert_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, 2U, + "id", yaml_hook_identity, + yaml_hook_canonical_item), + CBM_YAML_IDENTITY_EDIT_ERROR); + ASSERT_EQ(cbm_yaml_remove_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, 2U, + "id", yaml_hook_identity, + yaml_hook_canonical_item), + CBM_YAML_IDENTITY_EDIT_ERROR); + char *after = yaml_read_alloc(fixture.path); + ASSERT_NOT_NULL(after); + ASSERT_STR_EQ(after, cases[i]); + free(after); + th_cleanup(fixture.dir); + } + PASS(); +} + /* Uninstall must be idempotent against a config that was never written. Hermes * is detected by its binary on PATH, so a HOME without `.hermes/` reached the * YAML removers with an absent target — and they failed acquiring the lock @@ -2245,6 +2368,9 @@ SUITE(config_yaml_edit) { RUN_TEST(config_yaml_edit_nested_sequence_preserves_crlf); RUN_TEST(config_yaml_edit_nested_sequence_foreign_identity_is_preserved); RUN_TEST(config_yaml_edit_nested_sequence_removes_only_exact_canonical_item); + RUN_TEST(config_yaml_edit_nested_sequence_reinserts_under_emptied_parent); + RUN_TEST(config_yaml_edit_nested_sequence_expands_explicit_empty_parent_mapping); + RUN_TEST(config_yaml_edit_nested_sequence_still_refuses_non_empty_parent_flow); RUN_TEST(config_yaml_edit_nested_sequence_ignores_foreign_flow_and_block_scalars_issue2209); RUN_TEST(config_yaml_edit_nested_sequence_ambiguity_fails_byte_identically); RUN_TEST(config_yaml_edit_remove_on_absent_config_is_a_no_op); From 7a1bee25486b245b9fc6a75fd6d69c758bbb8a9d Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 3 Nov 2026 01:05:17 +0100 Subject: [PATCH 23/27] fix(index): keep the publication fence off raw allocator sites The memory-core ratchet (lint-memory-core) failed on the publication fence commit: it added six raw free() calls, each releasing a block that existing code allocates with raw calloc, malloc or strdup. cbm_free cannot release such a block without the core class counters drifting, so every new exit now reuses an existing release point instead: - index_supervisor.c: worker_artifacts_create() creates the response, log and publish fence files and names the one that failed, so all three failures share the single artifact_create_failed exit. - pipeline.c: a refused publication gate takes the existing source-changed abort exit (same CBM_PIPELINE_ABORT_PRESERVE_DB result and cleanup). - mcp.c: index_request_names_free() is the one release point for the request name strings, used by index_worker_json_result() and index_args_serving_index_servable(). Raw-site counts return to the origin/main values (index_supervisor.c 8, pipeline.c 129, mcp.c 765) with no baseline change. No behaviour change. Signed-off-by: Martin Vogel --- src/mcp/index_supervisor.c | 38 ++++++++++++++++++++++---------------- src/mcp/mcp.c | 20 +++++++++++++------- src/pipeline/pipeline.c | 17 +++++++---------- 3 files changed, 42 insertions(+), 33 deletions(-) diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index c8c4224ad0..701ac57436 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -1040,6 +1040,25 @@ static void worker_publish_fence_remove(cbm_index_worker_handle_t *handle) { } } +/* Create the private files of one attempt. NULL on success; otherwise the name + * of the artifact that could not be created, with errno as that step left it + * and every earlier artifact still in place for the caller to remove. */ +static const char *worker_artifacts_create(cbm_index_worker_handle_t *handle) { + if (!worker_unique_file(handle->response_path, sizeof(handle->response_path), "response")) { + return "response"; + } + if (!worker_unique_file(handle->log_path, sizeof(handle->log_path), "log")) { + return "log"; + } + /* Opt-in: only an enabled worker limit can stop a worker, so only then is + * there a publication to coordinate. */ + if (cbm_index_policy_worker_enabled(&handle->resource_policy) && + !worker_publish_fence_create(handle)) { + return "publish_fence"; + } + return NULL; +} + /* Settle who held the fence once the attempt is terminal. Claiming an unowned * fence now also bars any later claim, so a stray member of an unquiesced tree * can never publish after this report. */ @@ -1109,25 +1128,12 @@ static int worker_start_internal(const char *args_json, size_t memory_budget_byt handle->log_callback = log_callback; handle->log_context = log_context; worker_result_init(&handle->result); - if (!worker_unique_file(handle->response_path, sizeof(handle->response_path), "response") || - !worker_unique_file(handle->log_path, sizeof(handle->log_path), "log")) { + const char *failed_artifact = worker_artifacts_create(handle); + if (failed_artifact) { int saved_errno = errno; char error_text[CBM_SZ_32]; (void)snprintf(error_text, sizeof(error_text), "%d", saved_errno); - cbm_log_error("index.supervisor.artifact_create_failed", "artifact", - handle->response_path[0] ? "log" : "response", "errno", error_text); - (void)cbm_unlink(handle->response_path); - (void)cbm_unlink(handle->log_path); - free(handle); - return -1; - } - /* Opt-in: only an enabled worker limit can stop a worker, so only then is - * there a publication to coordinate. */ - if (cbm_index_policy_worker_enabled(&handle->resource_policy) && - !worker_publish_fence_create(handle)) { - char error_text[CBM_SZ_32]; - (void)snprintf(error_text, sizeof(error_text), "%d", errno); - cbm_log_error("index.supervisor.artifact_create_failed", "artifact", "publish_fence", + cbm_log_error("index.supervisor.artifact_create_failed", "artifact", failed_artifact, "errno", error_text); (void)cbm_unlink(handle->response_path); (void)cbm_unlink(handle->log_path); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index ddaba7e9dc..a8593d9466 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -11952,15 +11952,23 @@ static bool build_index_success_response(cbm_mcp_server_t *srv, yyjson_mut_doc * static bool project_db_is_servable(const char *project, const char *db_path); +/* The heap strings that name the project of an internal worker request + * (repo_path and name from cbm_mcp_get_string_arg, the project name derived from + * them). One release point for every helper that reads them; free(NULL) is a + * no-op, so any of them may be NULL. */ +static void index_request_names_free(char *repo_path, char *name_override, char *project_name) { + free(repo_path); + free(name_override); + free(project_name); +} + /* Serialize a worker JSON document and release the heap strings it borrowed. - * Extra arguments may be NULL; free(NULL) is a no-op. */ + * Extra arguments may be NULL. */ static char *index_worker_json_result(yyjson_mut_doc *doc, char *repo_path, char *name_override, char *project_name) { char *json = yy_doc_to_str(doc); yyjson_mut_doc_free(doc); - free(repo_path); - free(name_override); - free(project_name); + index_request_names_free(repo_path, name_override, project_name); char *result = cbm_mcp_text_result(json, true); free(json); return result; @@ -12034,9 +12042,7 @@ static bool index_args_serving_index_servable(const char *args) { project_db_path(project_name, db_path, sizeof(db_path)); servable = project_db_is_servable(project_name, db_path); } - free(repo_path); - free(name_override); - free(project_name); + index_request_names_free(repo_path, name_override, project_name); return servable; } diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index e393278ca6..519fc7d731 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -5240,19 +5240,16 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { cbm_userconfig_t *fresh_config = cbm_userconfig_load_with_source(p->repo_path); bool source_matches = cbm_pipeline_project_source_matches(p, fresh_config); cbm_userconfig_free(fresh_config); + bool publish_allowed = source_matches; if (!source_matches) { cbm_log_error("pipeline.err", "phase", "test_declarations_source_changed"); - cleanup_staging_db(staging_path); - free(staging_path); - free(final_path); - pipeline_release_test_config(p); - return CBM_PIPELINE_ABORT_PRESERVE_DB; - } - - /* Last point at which the serving database is still untouched: the - * destination preparation below may already move it aside. */ - if (p->publish_gate && !p->publish_gate(p->publish_gate_ctx)) { + } else if (p->publish_gate && !p->publish_gate(p->publish_gate_ctx)) { + /* Last point at which the serving database is still untouched: the + * destination preparation below may already move it aside. */ cbm_log_warn("pipeline.publish_refused", "path", final_path); + publish_allowed = false; + } + if (!publish_allowed) { cleanup_staging_db(staging_path); free(staging_path); free(final_path); From 0f1927fd3cf872aa3d85719339af255fb1890394 Mon Sep 17 00:00:00 2001 From: ngallodev Date: Sun, 8 Nov 2026 22:37:22 -0800 Subject: [PATCH 24/27] test(cli): run upstream Hermes hook cycle tests through the fork's hook installer Jenkins #91 failed two tests added by upstream #1862 follow-up e8f0f4a28: cli_hermes_hook_reinstalls_after_uninstall_empties_hooks and cli_hermes_hook_cycle_keeps_foreign_pre_llm_hook. Two fork differences caused it: - In this fork, `install` writes agent instructions and skills only; hooks come from `install-hooks`. The tests called cbm_install_agent_configs, which never writes the Hermes pre_llm_call hook here. They now call cbm_install_agent_hooks_for_testing, the same entry point the existing cli_hermes_stable_shell_context_contract test uses. - The fixture named the binary codebase-memory-mcp. The fork's uninstall identifies its owned hook by the codebase-memory-cli binary name, so it treated the upstream-named entry as user-modified and kept it. The fixture now uses codebase-memory-cli. The assertions are unchanged: one owned hook after install, `hooks: {}` accepted on uninstall/reinstall, and a foreign hook restored byte for byte. Locally all three Hermes hook tests pass. --- tests/test_cli.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/test_cli.c b/tests/test_cli.c index e16cad7bf4..7065ce0496 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -7456,9 +7456,9 @@ static bool cli_hermes_cycle_open(cli_hermes_cycle_t *cycle, const char *initial snprintf(cycle->config_path, sizeof(cycle->config_path), "%s/config.yaml", hermes_dir); #ifdef _WIN32 snprintf(cycle->binary_path, sizeof(cycle->binary_path), - "%s/.local/bin/codebase-memory-mcp.exe", cycle->tmpdir); + "%s/.local/bin/codebase-memory-cli.exe", cycle->tmpdir); #else - snprintf(cycle->binary_path, sizeof(cycle->binary_path), "%s/.local/bin/codebase-memory-mcp", + snprintf(cycle->binary_path, sizeof(cycle->binary_path), "%s/.local/bin/codebase-memory-cli", cycle->tmpdir); #endif test_mkdirp(hermes_dir); @@ -7490,13 +7490,15 @@ TEST(cli_hermes_hook_reinstalls_after_uninstall_empties_hooks) { FAIL("Hermes fake-HOME setup failed"); } char *argv[] = {"uninstall", "--yes"}; - int install_rc = cbm_install_agent_configs(cycle.tmpdir, cycle.binary_path, false, false); + int install_rc = + cbm_install_agent_hooks_for_testing(cycle.tmpdir, cycle.binary_path, false, false); char *installed = read_test_file_alloc(cycle.config_path); int uninstall_rc = cli_test_cmd_uninstall(2, argv); char *emptied = read_test_file_alloc(cycle.config_path); int repeat_rc = cli_test_cmd_uninstall(2, argv); char *repeated = read_test_file_alloc(cycle.config_path); - int reinstall_rc = cbm_install_agent_configs(cycle.tmpdir, cycle.binary_path, false, false); + int reinstall_rc = + cbm_install_agent_hooks_for_testing(cycle.tmpdir, cycle.binary_path, false, false); char *reinstalled = read_test_file_alloc(cycle.config_path); cli_hermes_cycle_close(&cycle); @@ -7539,11 +7541,13 @@ TEST(cli_hermes_hook_cycle_keeps_foreign_pre_llm_hook) { FAIL("Hermes fake-HOME setup failed"); } char *argv[] = {"uninstall", "--yes"}; - int install_rc = cbm_install_agent_configs(cycle.tmpdir, cycle.binary_path, false, false); + int install_rc = + cbm_install_agent_hooks_for_testing(cycle.tmpdir, cycle.binary_path, false, false); char *installed = read_test_file_alloc(cycle.config_path); int uninstall_rc = cli_test_cmd_uninstall(2, argv); char *restored = read_test_file_alloc(cycle.config_path); - int reinstall_rc = cbm_install_agent_configs(cycle.tmpdir, cycle.binary_path, false, false); + int reinstall_rc = + cbm_install_agent_hooks_for_testing(cycle.tmpdir, cycle.binary_path, false, false); char *reinstalled = read_test_file_alloc(cycle.config_path); cli_hermes_cycle_close(&cycle); From 96a1ac5a42bdbe48e5690f09e34824d4f576dd17 Mon Sep 17 00:00:00 2001 From: ngallodev Date: Mon, 9 Nov 2026 08:42:35 -0800 Subject: [PATCH 25/27] fix(subprocess): count worker descendants in their own process group in tree RSS Addresses CodeRabbit review on PR #34 (src/foundation/subprocess.c). The index worker's RSS probe summed only processes whose process group matched the worker's. Every child spawned through cbm_subprocess gets its own group (setpgid on Linux, POSIX_SPAWN_SETPGROUP on macOS), including `git ls-files -z` from discover.c. Discovery memory on a large Git index was therefore invisible to the index_max_rss_mb watchdog. Linux: the /proc scan now also reads ppid, collects processes into a bounded array (262144 entries, fails closed past it), and marks the worker group plus every descendant by parent pid (sorted + bsearch, repeated until no change). Each process is summed once; the existing unreadable-root, gone and empty outcomes are unchanged. macOS: the PROC_PGRP_ONLY listing is expanded with PROC_PPID_ONLY children of each collected pid, deduplicated, capped at 65536 and failing closed. Not compiled or run in this change; Linux only was verified. Windows is unchanged: the Job Object already covers descendants. Test: subprocess_tree_rss_counts_descendants_in_their_own_process_group spawns a real tree whose grandchild moves to its own group and holds 128 MiB, and asserts the measured total grows by at least 96 MiB. The __cbm_rss_tree_root helper role gains an optional `detach` argument; the detached holder exits when its parent disappears (alarm(120) backstop). --- src/foundation/subprocess.c | 185 ++++++++++++++++++++++++++++++++---- tests/test_main.c | 25 ++++- tests/test_subprocess.c | 34 ++++++- 3 files changed, 219 insertions(+), 25 deletions(-) diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index cadaab22c8..1c4d8e9db9 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -591,7 +591,7 @@ static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subproces * all-process scan cannot decide membership: proc_pidinfo answers EPERM for * every process of another user, so "not in our group" and "in our group but * unreadable" would look the same. Returns the member count, or -1. */ -static int cbm_tree_rss_list_group(pid_t pgid, pid_t **pids_out) { +static int cbm_tree_rss_list_pids(uint32_t type, uint32_t arg, pid_t **pids_out) { int capacity = 64; pid_t *pids = NULL; for (;;) { @@ -602,8 +602,7 @@ static int cbm_tree_rss_list_group(pid_t pgid, pid_t **pids_out) { } pids = grown; errno = 0; - int bytes = - proc_listpids(PROC_PGRP_ONLY, (uint32_t)pgid, pids, capacity * (int)sizeof(*pids)); + int bytes = proc_listpids(type, arg, pids, capacity * (int)sizeof(*pids)); if (bytes < 0 || (bytes == 0 && errno != 0)) { cbm_free(CBM_MEM_CLASS_OTHER, pids); return -1; @@ -621,6 +620,63 @@ static int cbm_tree_rss_list_group(pid_t pgid, pid_t **pids_out) { } } +/* Bound on the processes one probe retains; past it the probe fails closed. */ +enum { CBM_APPLE_PROBE_MAX_PROCS = 65536 }; + +/* The worker's process group plus every descendant by parent pid. Children + * placed in a group of their own (git discovery) are not in the group list. + * Pids already collected are skipped, so nothing is counted twice. Returns the + * count, or -1. */ +static int cbm_tree_rss_list_tree(pid_t pgid, pid_t **pids_out) { + pid_t *tree = NULL; + int count = cbm_tree_rss_list_pids(PROC_PGRP_ONLY, (uint32_t)pgid, &tree); + if (count < 0) { + return -1; + } + int capacity = count > 0 ? count : 1; + for (int index = 0; index < count; index++) { + pid_t *children = NULL; + int child_count = tree[index] > 0 ? cbm_tree_rss_list_pids(PROC_PPID_ONLY, + (uint32_t)tree[index], &children) + : 0; + if (child_count < 0) { + /* ESRCH-style races surface as an empty listing; a real error + * must not under-report the tree. */ + cbm_free(CBM_MEM_CLASS_OTHER, tree); + return -1; + } + for (int child = 0; child < child_count; child++) { + bool known = children[child] <= 0; + for (int seen = 0; !known && seen < count; seen++) { + known = tree[seen] == children[child]; + } + if (known) { + continue; + } + if (count >= CBM_APPLE_PROBE_MAX_PROCS) { + cbm_free(CBM_MEM_CLASS_OTHER, children); + cbm_free(CBM_MEM_CLASS_OTHER, tree); + return -1; + } + if (count == capacity) { + pid_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, tree, (size_t)capacity * 2 * sizeof(*tree)); + if (!grown) { + cbm_free(CBM_MEM_CLASS_OTHER, children); + cbm_free(CBM_MEM_CLASS_OTHER, tree); + return -1; + } + tree = grown; + capacity *= 2; + } + tree[count++] = children[child]; + } + cbm_free(CBM_MEM_CLASS_OTHER, children); + } + *pids_out = tree; + return count; +} + static int cbm_tree_rss_member_rusage(pid_t pid, bool root, struct rusage_info_v2 *usage) { #ifdef CBM_ENABLE_TEST_SEAMS int fault = cbm_tree_rss_member_fault(root); @@ -637,7 +693,7 @@ static int cbm_tree_rss_member_rusage(pid_t pid, bool root, struct rusage_info_v static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, uint64_t *rss_bytes) { pid_t *pids = NULL; - int count = cbm_tree_rss_list_group(process->pgid, &pids); + int count = cbm_tree_rss_list_tree(process->pgid, &pids); if (count < 0) { return CBM_PROC_TREE_RSS_ERROR; } @@ -730,8 +786,8 @@ static cbm_linux_stat_t cbm_linux_proc_stat_line(const char *pid_name, bool root return CBM_LINUX_STAT_OK; } -static cbm_linux_stat_t cbm_linux_proc_stat(const char *pid_name, bool root, pid_t *group, - int64_t *rss_pages) { +static cbm_linux_stat_t cbm_linux_proc_stat(const char *pid_name, bool root, pid_t *parent, + pid_t *group, int64_t *rss_pages) { char stat_line[4096]; cbm_linux_stat_t status = cbm_linux_proc_stat_line(pid_name, root, stat_line, sizeof(stat_line)); @@ -746,16 +802,20 @@ static cbm_linux_stat_t cbm_linux_proc_stat(const char *pid_name, bool root, pid char *save = NULL; char *token = strtok_r(command_end + 2, " ", &save); int field = 3; + bool have_parent = false; bool have_group = false; bool have_rss = false; while (token) { - if (field == 5 || field == 24) { + if (field == 4 || field == 5 || field == 24) { char *end = NULL; long long value = strtoll(token, &end, 10); if (!end || *end != '\0') { return CBM_LINUX_STAT_UNREADABLE; } - if (field == 5) { + if (field == 4) { + *parent = (pid_t)value; + have_parent = true; + } else if (field == 5) { *group = (pid_t)value; have_group = true; } else { @@ -767,7 +827,76 @@ static cbm_linux_stat_t cbm_linux_proc_stat(const char *pid_name, bool root, pid token = strtok_r(NULL, " ", &save); field++; } - return have_group && have_rss ? CBM_LINUX_STAT_OK : CBM_LINUX_STAT_UNREADABLE; + return have_parent && have_group && have_rss ? CBM_LINUX_STAT_OK : CBM_LINUX_STAT_UNREADABLE; +} + +/* One scanned process. `counted` is set once it is known to belong to the + * worker tree. */ +typedef struct { + pid_t pid; + pid_t parent; + pid_t group; + uint64_t bytes; + bool counted; +} cbm_linux_proc_t; + +/* Bound on the processes one probe retains. Past it the probe fails closed + * instead of reporting a total that may miss part of the tree. */ +enum { CBM_LINUX_PROBE_MAX_PROCS = 262144, CBM_LINUX_PROBE_INITIAL_PROCS = 512 }; + +static bool cbm_linux_probe_push(cbm_linux_proc_t **procs, size_t *count, size_t *capacity, + const cbm_linux_proc_t *proc) { + if (*count == *capacity) { + if (*capacity >= CBM_LINUX_PROBE_MAX_PROCS) { + return false; + } + size_t grown_capacity = *capacity ? *capacity * 2 : CBM_LINUX_PROBE_INITIAL_PROCS; + cbm_linux_proc_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, *procs, grown_capacity * sizeof(**procs)); + if (!grown) { + return false; + } + *procs = grown; + *capacity = grown_capacity; + } + (*procs)[(*count)++] = *proc; + return true; +} + +static int cbm_linux_probe_by_pid(const void *left, const void *right) { + pid_t a = ((const cbm_linux_proc_t *)left)->pid; + pid_t b = ((const cbm_linux_proc_t *)right)->pid; + return (a > b) - (a < b); +} + +/* Mark every process of the worker tree: members of its process group, plus + * any descendant by parent pid. Descendants can sit in a group of their own + * (git discovery children are spawned with setpgid), so the group alone would + * miss them. `procs` is sorted by pid, so a parent is found by binary search + * and each pass is O(n log n); passes repeat until no process is added (at + * most the tree depth). A process is counted once because the flag is checked + * before it is set. */ +static void cbm_linux_probe_mark_tree(cbm_linux_proc_t *procs, size_t count, pid_t pgid) { + qsort(procs, count, sizeof(*procs), cbm_linux_probe_by_pid); + for (size_t index = 0; index < count; index++) { + procs[index].counted = procs[index].group == pgid; + } + bool changed = true; + while (changed) { + changed = false; + for (size_t index = 0; index < count; index++) { + if (procs[index].counted || procs[index].parent <= 1) { + continue; + } + cbm_linux_proc_t key = {.pid = procs[index].parent}; + const cbm_linux_proc_t *parent = + bsearch(&key, procs, count, sizeof(*procs), cbm_linux_probe_by_pid); + if (parent && parent->counted) { + procs[index].counted = true; + changed = true; + } + } + } } static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, @@ -782,36 +911,54 @@ static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subproces return CBM_PROC_TREE_RSS_ERROR; } - uint64_t total = 0; - int measured = 0; + cbm_linux_proc_t *procs = NULL; + size_t count = 0; + size_t capacity = 0; bool failed = false; struct dirent *entry; while (!failed && (entry = readdir(proc)) != NULL) { if (!cbm_proc_pid_name(entry->d_name)) { continue; } + pid_t parent = 0; pid_t group = 0; int64_t pages = 0; - bool root_entry = strtol(entry->d_name, NULL, 10) == (long)process->pid; - cbm_linux_stat_t status = cbm_linux_proc_stat(entry->d_name, root_entry, &group, &pages); + long pid = strtol(entry->d_name, NULL, 10); + bool root_entry = pid == (long)process->pid; + cbm_linux_stat_t status = + cbm_linux_proc_stat(entry->d_name, root_entry, &parent, &group, &pages); if (status != CBM_LINUX_STAT_OK) { failed = root_entry || status != CBM_LINUX_STAT_GONE; continue; } - if (group != process->pgid || pages < 0) { + if (pages < 0) { continue; } uint64_t page_count = (uint64_t)pages; - uint64_t bytes = page_count > UINT64_MAX / (uint64_t)page_size - ? UINT64_MAX - : page_count * (uint64_t)page_size; - cbm_rss_add_saturated(&total, bytes); - measured++; + cbm_linux_proc_t item = {.pid = (pid_t)pid, + .parent = parent, + .group = group, + .bytes = page_count > UINT64_MAX / (uint64_t)page_size + ? UINT64_MAX + : page_count * (uint64_t)page_size}; + failed = !cbm_linux_probe_push(&procs, &count, &capacity, &item); } (void)closedir(proc); if (failed) { + cbm_free(CBM_MEM_CLASS_OTHER, procs); return CBM_PROC_TREE_RSS_ERROR; } + + cbm_linux_probe_mark_tree(procs, count, process->pgid); + uint64_t total = 0; + int measured = 0; + for (size_t index = 0; index < count; index++) { + if (procs[index].counted) { + cbm_rss_add_saturated(&total, procs[index].bytes); + measured++; + } + } + cbm_free(CBM_MEM_CLASS_OTHER, procs); if (measured == 0) { return CBM_PROC_TREE_RSS_EMPTY; } diff --git a/tests/test_main.c b/tests/test_main.c index 63fe5a2ccb..0f36c9a7c3 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -2044,7 +2044,7 @@ static int tf_role_env_isolation(int argc, char **argv) { * publishes its pid by an atomic rename once every page is touched, then * blocks until it is killed. The root exits when its descendant dies, so a * failed setup ends the tree instead of leaving the test waiting. */ -static int tf_rss_tree_hold(const char *ready_path, const char *mib_text) { +static int tf_rss_tree_hold(const char *ready_path, const char *mib_text, bool own_group) { char *end = NULL; long mib = strtol(mib_text, &end, 10); if (!end || *end != '\0' || mib < 0 || mib > 1024) { @@ -2074,6 +2074,18 @@ static int tf_rss_tree_hold(const char *ready_path, const char *mib_text) { } volatile unsigned char keep = block ? block[bytes - 1] : 0; (void)keep; +#ifndef _WIN32 + if (own_group) { + /* Nothing kills a process outside the worker's group, so leave when + * the root (the parent) is gone, with an alarm as a backstop. */ + pid_t parent = getppid(); + (void)alarm(120); + while (getppid() == parent) { + cbm_usleep(10000); + } + return 0; + } +#endif for (;;) { #ifdef _WIN32 Sleep(INFINITE); @@ -2084,11 +2096,13 @@ static int tf_rss_tree_hold(const char *ready_path, const char *mib_text) { } static int tf_role_rss_tree_hold(int argc, char **argv) { - return argc == 4 ? tf_rss_tree_hold(argv[2], argv[3]) : -1; + return argc == 4 ? tf_rss_tree_hold(argv[2], argv[3], false) : -1; } static int tf_role_rss_tree_root(int argc, char **argv) { - if (argc != 4) { + /* A fifth argument "detach" puts the descendant in a process group of its + * own, as git discovery children are. */ + if (argc != 4 && argc != 5) { return -1; } #ifdef _WIN32 @@ -2123,7 +2137,10 @@ static int tf_role_rss_tree_root(int argc, char **argv) { return 85; } if (child == 0) { - _exit(tf_rss_tree_hold(argv[2], argv[3])); + if (argc == 5 && setpgid(0, 0) != 0) { + _exit(86); + } + _exit(tf_rss_tree_hold(argv[2], argv[3], argc == 5)); } int status = 0; while (waitpid(child, &status, 0) < 0 && errno == EINTR) {} diff --git a/tests/test_subprocess.c b/tests/test_subprocess.c index 6bec175c82..2ae369e85f 100644 --- a/tests/test_subprocess.c +++ b/tests/test_subprocess.c @@ -885,7 +885,7 @@ static bool rss_tree_finish(cbm_subprocess_t *process, rss_tree_sample_t *sample /* Measure one tree whose descendant holds `mib` MiB, with every non-root * member's measurement failing with `member_fault` (0 = real calls). */ -static rss_tree_sample_t rss_tree_sample(int mib, int member_fault) { +static rss_tree_sample_t rss_tree_sample_in(int mib, int member_fault, bool own_group) { rss_tree_sample_t sample = {.status = CBM_PROC_TREE_RSS_ERROR, .final_status = CBM_PROC_TREE_RSS_ERROR, .final_rss = UINT64_MAX}; @@ -898,7 +898,12 @@ static rss_tree_sample_t rss_tree_sample(int mib, int member_fault) { return sample; } (void)snprintf(mib_text, sizeof(mib_text), "%d", mib); - const char *argv[] = {cbm_test_program_path, "__cbm_rss_tree_root", ready_path, mib_text, NULL}; + const char *argv[] = {cbm_test_program_path, + "__cbm_rss_tree_root", + ready_path, + mib_text, + own_group ? "detach" : NULL, + NULL}; cbm_proc_opts_t opts = {0}; opts.bin = cbm_test_program_path; opts.argv = argv; @@ -918,6 +923,10 @@ static rss_tree_sample_t rss_tree_sample(int mib, int member_fault) { return sample; } +static rss_tree_sample_t rss_tree_sample(int mib, int member_fault) { + return rss_tree_sample_in(mib, member_fault, false); +} + /* Only the DESCENDANT allocates, so the difference between two otherwise * identical trees is memory the probe can only have counted by measuring past * the root. 128 MiB touched against a 96 MiB floor leaves room for allocator @@ -945,6 +954,26 @@ TEST(subprocess_tree_rss_measures_contained_descendants) { PASS(); } +/* git discovery children run in a process group of their own. The probe must + * still count them: the memory is the worker's, and a group-only probe would + * let it grow past the limit unseen. POSIX only (a Job Object has no groups). */ +TEST(subprocess_tree_rss_counts_descendants_in_their_own_process_group) { +#ifdef _WIN32 + SKIP_PLATFORM("process groups are POSIX; the Job Object probe already covers descendants"); +#else + rss_tree_sample_t idle = rss_tree_sample_in(0, 0, true); + rss_tree_sample_t loaded = rss_tree_sample_in(128, 0, true); + ASSERT_TRUE(idle.ready); + ASSERT_EQ(idle.status, CBM_PROC_TREE_RSS_OK); + ASSERT_TRUE(loaded.ready); + ASSERT_EQ(loaded.status, CBM_PROC_TREE_RSS_OK); + ASSERT_TRUE(loaded.rss >= idle.rss + 96 * RSS_TREE_MIB); + ASSERT_TRUE(idle.quiesced); + ASSERT_TRUE(loaded.quiesced); + PASS(); +#endif +} + /* A live member the platform will not let us measure must fail the probe: a * total without it would under-report the tree. */ TEST(subprocess_tree_rss_fails_when_a_live_member_cannot_be_measured) { @@ -2817,6 +2846,7 @@ SUITE(subprocess) { RUN_TEST(subprocess_run_null_bin_rejected); RUN_TEST(subprocess_spawn_returns_while_child_is_running); RUN_TEST(subprocess_tree_rss_measures_contained_descendants); + RUN_TEST(subprocess_tree_rss_counts_descendants_in_their_own_process_group); RUN_TEST(subprocess_tree_rss_fails_when_a_live_member_cannot_be_measured); RUN_TEST(subprocess_tree_rss_skips_a_member_that_exited_mid_probe); RUN_TEST(subprocess_natural_completion_is_cached_across_polls); From e5a4dc0d4a5cf1fda9c6f08155f13560e4bbf07f Mon Sep 17 00:00:00 2001 From: ngallodev Date: Mon, 9 Nov 2026 08:42:36 -0800 Subject: [PATCH 26/27] fix(php): keep PHP after comment-ended close tags, CR comments and bare short tags Addresses CodeRabbit review on PR #34 (internal/cbm/php_inline_html.c and two test findings). Inline-HTML masker: - Line comments now end at a lone CR as well as LF, so CR-only files do not scan into the next line and treat `?>` inside a string as a close tag. - A `?>` that ends a `//` or `#` comment used to become `; ` with the comment still open, hiding PHP reopened on the same line. The comment bytes up to the close tag are now blanked to spaces. Byte offsets and rows are unchanged; only that comment's text is lost to the parser. - With short tags, `` prolog in markup stays markup. */ + * and `` that ends the one-line comment at i. */ static int php_skip_line_comment(const char *s, int len, int i) { - while (i < len && s[i] != '\n' && !php_at(s, len, i, "?>")) { + while (i < len && s[i] != '\n' && s[i] != '\r' && !php_at(s, len, i, "?>")) { i++; } return i; @@ -236,7 +248,8 @@ static int php_skip_ident(const char *s, int len, int i, bool *halt) { /* Scan PHP code from i to the next close tag. Returns the index of that `?>`, * len when the file ends in PHP mode, or PHP_NOT_FOUND at __halt_compiler. */ -static int php_scan_code(const char *s, int len, int i) { +static int php_scan_code(const char *s, int len, int i, int *comment) { + *comment = PHP_NOT_FOUND; while (i < len) { unsigned char c = (unsigned char)s[i]; if (php_at(s, len, i, "?>")) { @@ -247,7 +260,11 @@ static int php_scan_code(const char *s, int len, int i) { } else if (php_at(s, len, i, "/*")) { i = php_skip_block_comment(s, len, i); } else if (php_at(s, len, i, "//") || (c == '#' && !php_at(s, len, i, "#["))) { + int start = i; i = php_skip_line_comment(s, len, i); + if (php_at(s, len, i, "?>")) { + *comment = start; /* the close tag ends this comment */ + } } else if (php_at(s, len, i, "<<<")) { i = php_skip_heredoc(s, len, i); } else if (php_is_ident_start(c)) { @@ -295,10 +312,15 @@ static void php_mask_blank(PHPMask *m, int from, int to) { } } -/* Turn the `?>` at i into `; `. */ -static void php_mask_close_tag(PHPMask *m, int i) { +/* Turn the `?>` at i into `; `. When the tag ends a line comment starting at + * `comment`, blank that comment first: `; ` cannot end it and a line break + * would shift rows, so the reopened PHP on the same line would be swallowed. */ +static void php_mask_close_tag(PHPMask *m, int i, int comment) { char *out = php_mask_writable(m); if (out) { + if (comment >= 0) { + php_blank(out, comment, i); + } out[i] = ';'; out[i + SKIP_ONE] = ' '; } @@ -336,11 +358,12 @@ const char *cbm_php_mask_inline_html(CBMArena *arena, const char *source, int so seen_open = true; /* PHP mode, up to the close tag. */ - int close = php_scan_code(source, source_len, i + tag); + int comment = PHP_NOT_FOUND; + int close = php_scan_code(source, source_len, i + tag, &comment); if (close < 0 || close >= source_len) { break; } - php_mask_close_tag(&m, close); + php_mask_close_tag(&m, close, comment); i = close + PAIR_LEN; } /* On allocation failure parse the file as it is: the old behaviour. */ diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index adf7495788..ce7a07f2a3 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -2774,7 +2774,9 @@ TEST(daemon_application_programmatic_index_forwards_set_doc_candidates) { } TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { - char *cache = th_mktempdir("cbm_app_worker_resource_cache"); + /* th_mktempdir returns one static buffer: copy the cache path. */ + char *cache_made = th_mktempdir("cbm_app_worker_resource_cache"); + char *cache = cache_made ? strdup(cache_made) : NULL; cbm_config_t *stored_config = cache ? cbm_config_open(cache) : NULL; app_fake_worker_context_t fake; app_fake_worker_context_init(&fake); @@ -2799,7 +2801,8 @@ TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { cbm_daemon_runtime_application_callbacks_t callbacks = cbm_daemon_application_runtime_callbacks(application); cbm_daemon_runtime_application_session_t *session = app_test_open(&callbacks, 4188); - char *root = th_mktempdir("cbm_app_worker_resource"); + char *root_made = th_mktempdir("cbm_app_worker_resource"); + char *root = root_made ? strdup(root_made) : NULL; uint8_t *context = NULL; uint32_t context_length = 0; uint8_t *tool = NULL; @@ -2847,6 +2850,8 @@ TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { free(tool); th_cleanup(root); th_cleanup(cache); + free(root); + free(cache); ASSERT_TRUE(setup); ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_OK); @@ -2858,7 +2863,9 @@ TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { } TEST(daemon_application_worker_resource_failure_after_crash_stops_recovery) { - char *cache = th_mktempdir("cbm_app_worker_resource_crash_cache"); + /* th_mktempdir returns one static buffer: copy the cache path. */ + char *cache_made = th_mktempdir("cbm_app_worker_resource_crash_cache"); + char *cache = cache_made ? strdup(cache_made) : NULL; cbm_config_t *stored_config = cache ? cbm_config_open(cache) : NULL; app_fake_worker_context_t fake; app_fake_worker_context_init(&fake); @@ -2887,7 +2894,8 @@ TEST(daemon_application_worker_resource_failure_after_crash_stops_recovery) { cbm_daemon_runtime_application_callbacks_t callbacks = cbm_daemon_application_runtime_callbacks(application); cbm_daemon_runtime_application_session_t *session = app_test_open(&callbacks, 4189); - char *root = th_mktempdir("cbm_app_worker_resource_crash"); + char *root_made = th_mktempdir("cbm_app_worker_resource_crash"); + char *root = root_made ? strdup(root_made) : NULL; uint8_t *context = NULL; uint32_t context_length = 0; uint8_t *tool = NULL; @@ -2935,6 +2943,8 @@ TEST(daemon_application_worker_resource_failure_after_crash_stops_recovery) { free(tool); th_cleanup(root); th_cleanup(cache); + free(root); + free(cache); ASSERT_TRUE(setup); ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_OK); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index e475e1ff9b..215800a0a4 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1110,6 +1110,37 @@ TEST(php_inline_html_trailing_close_tag_crlf_issue2000) { PASS(); } +/* Standalone CR ends a line comment, so a `?>` inside the next CR-only line's + * string is not a close tag; a `?>` that ends a `//` or `#` comment blanks the + * comment so the PHP reopened on that line is not swallowed; `';\rfunction a() {}\r"; + ASSERT(cbm_php_mask_inline_html(&arena, cr, (int)strlen(cr)) == cr); + + const char *cmt = "\n\n"; + masked = cbm_php_mask_inline_html(&arena, shorttag, (int)strlen(shorttag)); + ASSERT_STR_EQ(masked, "init_usages = 0; + o->init_usages = -1; /* error sentinel until the USAGE query succeeds */ if (cbm_store_find_edges_by_type(s, project, "USAGE", &edges, &count) == CBM_STORE_OK) { + o->init_usages = 0; for (int i = 0; i < count; i++) { cbm_node_t tgt = {0}; if (cbm_store_find_node_by_id(s, edges[i].target_id, &tgt) == CBM_STORE_OK && @@ -23148,6 +23149,7 @@ static void swift_init_case(bool parallel, int pad, SwiftInitObservation *o) { memset(o, 0, sizeof(*o)); o->run_rc = -1; o->calls = -1; + o->init_usages = -1; char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_swift_init_XXXXXX"); if (!cbm_mkdtemp(tmp)) { From f2b0dae5440d534dc4df8cc0a518007213cde39f Mon Sep 17 00:00:00 2001 From: ngallodev Date: Mon, 9 Nov 2026 09:19:06 -0800 Subject: [PATCH 27/27] fix(subprocess): return EMPTY before marking when the /proc scan collected nothing Jenkins #95 failed in lint-cppcheck (ctunullpointer): when readdir returned no usable entries, procs stayed NULL and was passed to cbm_linux_probe_mark_tree, which calls qsort on it. A zero count made that harmless at runtime, but passing NULL to qsort is undefined. The probe now returns CBM_PROC_TREE_RSS_EMPTY directly when nothing was collected, matching the result the loop would have produced. `scripts/lint.sh --ci` and the subprocess suite (61 pass) pass locally. --- src/foundation/subprocess.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 1c4d8e9db9..37360d8315 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -948,6 +948,10 @@ static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subproces cbm_free(CBM_MEM_CLASS_OTHER, procs); return CBM_PROC_TREE_RSS_ERROR; } + if (!procs || count == 0) { + cbm_free(CBM_MEM_CLASS_OTHER, procs); + return CBM_PROC_TREE_RSS_EMPTY; + } cbm_linux_probe_mark_tree(procs, count, process->pgid); uint64_t total = 0;